1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ 3 #define _GNU_SOURCE 4 #include <argp.h> 5 #include <libgen.h> 6 #include <ctype.h> 7 #include <string.h> 8 #include <stdlib.h> 9 #include <sched.h> 10 #include <pthread.h> 11 #include <dirent.h> 12 #include <signal.h> 13 #include <fcntl.h> 14 #include <unistd.h> 15 #include <sys/time.h> 16 #include <sys/sysinfo.h> 17 #include <sys/stat.h> 18 #include <bpf/libbpf.h> 19 #include <bpf/btf.h> 20 #include <bpf/bpf.h> 21 #include <libelf.h> 22 #include <gelf.h> 23 #include <float.h> 24 #include <math.h> 25 #include <limits.h> 26 #include <assert.h> 27 28 #ifndef ARRAY_SIZE 29 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) 30 #endif 31 32 #ifndef max 33 #define max(a, b) ((a) > (b) ? (a) : (b)) 34 #endif 35 36 #ifndef min 37 #define min(a, b) ((a) < (b) ? (a) : (b)) 38 #endif 39 40 enum stat_id { 41 VERDICT, 42 DURATION, 43 TOTAL_INSNS, 44 TOTAL_STATES, 45 PEAK_STATES, 46 MAX_STATES_PER_INSN, 47 MARK_READ_MAX_LEN, 48 SIZE, 49 JITED_SIZE, 50 STACK, 51 PROG_TYPE, 52 ATTACH_TYPE, 53 MEMORY_PEAK, 54 55 FILE_NAME, 56 PROG_NAME, 57 58 ALL_STATS_CNT, 59 NUM_STATS_CNT = FILE_NAME - VERDICT, 60 }; 61 62 /* In comparison mode each stat can specify up to four different values: 63 * - A side value; 64 * - B side value; 65 * - absolute diff value; 66 * - relative (percentage) diff value. 67 * 68 * When specifying stat specs in comparison mode, user can use one of the 69 * following variant suffixes to specify which exact variant should be used for 70 * ordering or filtering: 71 * - `_a` for A side value; 72 * - `_b` for B side value; 73 * - `_diff` for absolute diff value; 74 * - `_pct` for relative (percentage) diff value. 75 * 76 * If no variant suffix is provided, then `_b` (control data) is assumed. 77 * 78 * As an example, let's say instructions stat has the following output: 79 * 80 * Insns (A) Insns (B) Insns (DIFF) 81 * --------- --------- -------------- 82 * 21547 20920 -627 (-2.91%) 83 * 84 * Then: 85 * - 21547 is A side value (insns_a); 86 * - 20920 is B side value (insns_b); 87 * - -627 is absolute diff value (insns_diff); 88 * - -2.91% is relative diff value (insns_pct). 89 * 90 * For verdict there is no verdict_pct variant. 91 * For file and program name, _a and _b variants are equivalent and there are 92 * no _diff or _pct variants. 93 */ 94 enum stat_variant { 95 VARIANT_A, 96 VARIANT_B, 97 VARIANT_DIFF, 98 VARIANT_PCT, 99 }; 100 101 struct verif_stats { 102 char *file_name; 103 char *prog_name; 104 105 long stats[NUM_STATS_CNT]; 106 }; 107 108 /* joined comparison mode stats */ 109 struct verif_stats_join { 110 char *file_name; 111 char *prog_name; 112 113 const struct verif_stats *stats_a; 114 const struct verif_stats *stats_b; 115 }; 116 117 struct stat_specs { 118 int spec_cnt; 119 enum stat_id ids[ALL_STATS_CNT]; 120 enum stat_variant variants[ALL_STATS_CNT]; 121 bool asc[ALL_STATS_CNT]; 122 bool abs[ALL_STATS_CNT]; 123 int lens[ALL_STATS_CNT * 3]; /* 3x for comparison mode */ 124 }; 125 126 enum resfmt { 127 RESFMT_TABLE, 128 RESFMT_TABLE_CALCLEN, /* fake format to pre-calculate table's column widths */ 129 RESFMT_CSV, 130 }; 131 132 enum filter_kind { 133 FILTER_NAME, 134 FILTER_STAT, 135 }; 136 137 enum operator_kind { 138 OP_EQ, /* == or = */ 139 OP_NEQ, /* != or <> */ 140 OP_LT, /* < */ 141 OP_LE, /* <= */ 142 OP_GT, /* > */ 143 OP_GE, /* >= */ 144 }; 145 146 struct filter { 147 enum filter_kind kind; 148 /* FILTER_NAME */ 149 char *any_glob; 150 char *file_glob; 151 char *prog_glob; 152 /* FILTER_STAT */ 153 enum operator_kind op; 154 int stat_id; 155 enum stat_variant stat_var; 156 long value; 157 bool abs; 158 }; 159 160 struct rvalue { 161 enum { INTEGRAL, ENUMERATOR } type; 162 union { 163 long long ivalue; 164 char *svalue; 165 }; 166 }; 167 168 struct field_access { 169 enum { FIELD_NAME, ARRAY_INDEX } type; 170 union { 171 char *name; 172 struct rvalue index; 173 }; 174 }; 175 176 struct var_preset { 177 struct field_access *atoms; 178 int atom_count; 179 char *full_name; 180 struct rvalue value; 181 bool applied; 182 }; 183 184 enum dump_mode { 185 DUMP_NONE = 0, 186 DUMP_XLATED = 1, 187 DUMP_JITED = 2, 188 }; 189 190 static struct env { 191 char **filenames; 192 int filename_cnt; 193 bool verbose; 194 bool debug; 195 bool quiet; 196 bool force_checkpoints; 197 bool force_reg_invariants; 198 enum resfmt out_fmt; 199 bool show_version; 200 bool comparison_mode; 201 bool replay_mode; 202 int top_n; 203 204 int log_level; 205 int log_size; 206 bool log_fixed; 207 208 struct verif_stats *prog_stats; 209 int prog_stat_cnt; 210 211 /* baseline_stats is allocated and used only in comparison mode */ 212 struct verif_stats *baseline_stats; 213 int baseline_stat_cnt; 214 215 struct verif_stats_join *join_stats; 216 int join_stat_cnt; 217 218 struct stat_specs output_spec; 219 struct stat_specs sort_spec; 220 221 struct filter *allow_filters; 222 struct filter *deny_filters; 223 int allow_filter_cnt; 224 int deny_filter_cnt; 225 226 int files_processed; 227 int files_skipped; 228 int progs_processed; 229 int progs_skipped; 230 int top_src_lines; 231 struct var_preset *presets; 232 int npresets; 233 char orig_cgroup[PATH_MAX]; 234 char stat_cgroup[PATH_MAX]; 235 int memory_peak_fd; 236 __u32 dump_mode; 237 } env; 238 239 static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args) 240 { 241 if (!env.verbose) 242 return 0; 243 if (level == LIBBPF_DEBUG && !env.debug) 244 return 0; 245 return vfprintf(stderr, format, args); 246 } 247 248 #define log_errno(fmt, ...) log_errno_aux(__FILE__, __LINE__, fmt, ##__VA_ARGS__) 249 250 __attribute__((format(printf, 3, 4))) 251 static int log_errno_aux(const char *file, int line, const char *fmt, ...) 252 { 253 int err = -errno; 254 va_list ap; 255 256 va_start(ap, fmt); 257 fprintf(stderr, "%s:%d: ", file, line); 258 vfprintf(stderr, fmt, ap); 259 fprintf(stderr, " failed with error '%s'.\n", strerror(errno)); 260 va_end(ap); 261 return err; 262 } 263 264 #ifndef VERISTAT_VERSION 265 #define VERISTAT_VERSION "<kernel>" 266 #endif 267 268 const char *argp_program_version = "veristat v" VERISTAT_VERSION; 269 const char *argp_program_bug_address = "<bpf@vger.kernel.org>"; 270 const char argp_program_doc[] = 271 "veristat BPF verifier stats collection and comparison tool.\n" 272 "\n" 273 "USAGE: veristat <obj-file> [<obj-file>...]\n" 274 " OR: veristat -C <baseline.csv> <comparison.csv>\n" 275 " OR: veristat -R <results.csv>\n" 276 " OR: veristat -vl2 <to_analyze.bpf.o>\n"; 277 278 enum { 279 OPT_LOG_FIXED = 1000, 280 OPT_LOG_SIZE = 1001, 281 OPT_DUMP = 1002, 282 }; 283 284 static const struct argp_option opts[] = { 285 { NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help" }, 286 { "version", 'V', NULL, 0, "Print version" }, 287 { "verbose", 'v', NULL, 0, "Verbose mode" }, 288 { "debug", 'd', NULL, 0, "Debug mode (turns on libbpf debug logging)" }, 289 { "log-level", 'l', "LEVEL", 0, "Verifier log level (default 0 for normal mode, 1 for verbose mode, 2 for full verification log)" }, 290 { "log-fixed", OPT_LOG_FIXED, NULL, 0, "Disable verifier log rotation" }, 291 { "log-size", OPT_LOG_SIZE, "BYTES", 0, "Customize verifier log size (default to 16MB)" }, 292 { "top-n", 'n', "N", 0, "Emit only up to first N results." }, 293 { "quiet", 'q', NULL, 0, "Quiet mode" }, 294 { "emit", 'e', "SPEC", 0, "Specify stats to be emitted" }, 295 { "sort", 's', "SPEC", 0, "Specify sort order" }, 296 { "output-format", 'o', "FMT", 0, "Result output format (table, csv), default is table." }, 297 { "compare", 'C', NULL, 0, "Comparison mode" }, 298 { "replay", 'R', NULL, 0, "Replay mode" }, 299 { "filter", 'f', "FILTER", 0, "Filter expressions (or @filename for file with expressions)." }, 300 { "test-states", 't', NULL, 0, 301 "Force frequent BPF verifier state checkpointing (set BPF_F_TEST_STATE_FREQ program flag)" }, 302 { "test-reg-invariants", 'r', NULL, 0, 303 "Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" }, 304 { "top-src-lines", 'S', "N", 0, "Emit N most frequent source code lines" }, 305 { "set-global-vars", 'G', "GLOBAL", 0, "Set global variables provided in the expression, for example \"var1 = 1\"" }, 306 { "dump", OPT_DUMP, "DUMP_MODE", OPTION_ARG_OPTIONAL, "Print BPF program dump (xlated, jited)" }, 307 {}, 308 }; 309 310 static int parse_stats(const char *stats_str, struct stat_specs *specs); 311 static int append_filter(struct filter **filters, int *cnt, const char *str); 312 static int append_filter_file(const char *path); 313 static int append_var_preset(struct var_preset **presets, int *cnt, const char *expr); 314 static int append_var_preset_file(const char *filename); 315 static int append_file(const char *path); 316 static int append_file_from_file(const char *path); 317 318 static error_t parse_arg(int key, char *arg, struct argp_state *state) 319 { 320 int err; 321 322 switch (key) { 323 case 'h': 324 argp_state_help(state, stderr, ARGP_HELP_STD_HELP); 325 break; 326 case 'V': 327 env.show_version = true; 328 break; 329 case 'v': 330 env.verbose = true; 331 break; 332 case 'd': 333 env.debug = true; 334 env.verbose = true; 335 break; 336 case 'q': 337 env.quiet = true; 338 break; 339 case 'e': 340 err = parse_stats(arg, &env.output_spec); 341 if (err) 342 return err; 343 break; 344 case 's': 345 err = parse_stats(arg, &env.sort_spec); 346 if (err) 347 return err; 348 break; 349 case 'o': 350 if (strcmp(arg, "table") == 0) { 351 env.out_fmt = RESFMT_TABLE; 352 } else if (strcmp(arg, "csv") == 0) { 353 env.out_fmt = RESFMT_CSV; 354 } else { 355 fprintf(stderr, "Unrecognized output format '%s'\n", arg); 356 return -EINVAL; 357 } 358 break; 359 case 'l': 360 errno = 0; 361 env.log_level = strtol(arg, NULL, 10); 362 if (errno) { 363 fprintf(stderr, "invalid log level: %s\n", arg); 364 argp_usage(state); 365 } 366 break; 367 case OPT_LOG_FIXED: 368 env.log_fixed = true; 369 break; 370 case OPT_LOG_SIZE: 371 errno = 0; 372 env.log_size = strtol(arg, NULL, 10); 373 if (errno) { 374 fprintf(stderr, "invalid log size: %s\n", arg); 375 argp_usage(state); 376 } 377 break; 378 case 't': 379 env.force_checkpoints = true; 380 break; 381 case 'r': 382 env.force_reg_invariants = true; 383 break; 384 case 'n': 385 errno = 0; 386 env.top_n = strtol(arg, NULL, 10); 387 if (errno) { 388 fprintf(stderr, "invalid top N specifier: %s\n", arg); 389 argp_usage(state); 390 } 391 break; 392 case 'C': 393 env.comparison_mode = true; 394 break; 395 case 'R': 396 env.replay_mode = true; 397 break; 398 case 'f': 399 if (arg[0] == '@') 400 err = append_filter_file(arg + 1); 401 else if (arg[0] == '!') 402 err = append_filter(&env.deny_filters, &env.deny_filter_cnt, arg + 1); 403 else 404 err = append_filter(&env.allow_filters, &env.allow_filter_cnt, arg); 405 if (err) { 406 fprintf(stderr, "Failed to collect program filter expressions: %d\n", err); 407 return err; 408 } 409 break; 410 case 'S': 411 errno = 0; 412 env.top_src_lines = strtol(arg, NULL, 10); 413 if (errno) { 414 fprintf(stderr, "invalid top lines N specifier: %s\n", arg); 415 argp_usage(state); 416 } 417 break; 418 case 'G': { 419 if (arg[0] == '@') 420 err = append_var_preset_file(arg + 1); 421 else 422 err = append_var_preset(&env.presets, &env.npresets, arg); 423 if (err) { 424 fprintf(stderr, "Failed to parse global variable presets: %s\n", arg); 425 return err; 426 } 427 break; 428 } 429 case ARGP_KEY_ARG: 430 if (arg[0] == '@') 431 err = append_file_from_file(arg + 1); 432 else 433 err = append_file(arg); 434 if (err) { 435 fprintf(stderr, "Failed to collect BPF object files: %d\n", err); 436 return err; 437 } 438 break; 439 case OPT_DUMP: 440 if (!arg || strcasecmp(arg, "xlated") == 0) { 441 env.dump_mode |= DUMP_XLATED; 442 } else if (strcasecmp(arg, "jited") == 0) { 443 env.dump_mode |= DUMP_JITED; 444 } else { 445 fprintf(stderr, "Unrecognized dump mode '%s'\n", arg); 446 return -EINVAL; 447 } 448 break; 449 default: 450 return ARGP_ERR_UNKNOWN; 451 } 452 return 0; 453 } 454 455 static const struct argp argp = { 456 .options = opts, 457 .parser = parse_arg, 458 .doc = argp_program_doc, 459 }; 460 461 462 /* Adapted from perf/util/string.c */ 463 static bool glob_matches(const char *str, const char *pat) 464 { 465 while (*str && *pat && *pat != '*') { 466 if (*str != *pat) 467 return false; 468 str++; 469 pat++; 470 } 471 /* Check wild card */ 472 if (*pat == '*') { 473 while (*pat == '*') 474 pat++; 475 if (!*pat) /* Tail wild card matches all */ 476 return true; 477 while (*str) 478 if (glob_matches(str++, pat)) 479 return true; 480 } 481 return !*str && !*pat; 482 } 483 484 static bool is_bpf_obj_file(const char *path) { 485 Elf64_Ehdr *ehdr; 486 int fd, err = -EINVAL; 487 Elf *elf = NULL; 488 489 fd = open(path, O_RDONLY | O_CLOEXEC); 490 if (fd < 0) 491 return true; /* we'll fail later and propagate error */ 492 493 /* ensure libelf is initialized */ 494 (void)elf_version(EV_CURRENT); 495 496 elf = elf_begin(fd, ELF_C_READ, NULL); 497 if (!elf) 498 goto cleanup; 499 500 if (elf_kind(elf) != ELF_K_ELF || gelf_getclass(elf) != ELFCLASS64) 501 goto cleanup; 502 503 ehdr = elf64_getehdr(elf); 504 /* Old LLVM set e_machine to EM_NONE */ 505 if (!ehdr || ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) 506 goto cleanup; 507 508 err = 0; 509 cleanup: 510 if (elf) 511 elf_end(elf); 512 close(fd); 513 return err == 0; 514 } 515 516 static bool should_process_file_prog(const char *filename, const char *prog_name) 517 { 518 struct filter *f; 519 int i, allow_cnt = 0; 520 521 for (i = 0; i < env.deny_filter_cnt; i++) { 522 f = &env.deny_filters[i]; 523 if (f->kind != FILTER_NAME) 524 continue; 525 526 if (f->any_glob && glob_matches(filename, f->any_glob)) 527 return false; 528 if (f->any_glob && prog_name && glob_matches(prog_name, f->any_glob)) 529 return false; 530 if (f->file_glob && glob_matches(filename, f->file_glob)) 531 return false; 532 if (f->prog_glob && prog_name && glob_matches(prog_name, f->prog_glob)) 533 return false; 534 } 535 536 for (i = 0; i < env.allow_filter_cnt; i++) { 537 f = &env.allow_filters[i]; 538 if (f->kind != FILTER_NAME) 539 continue; 540 541 allow_cnt++; 542 if (f->any_glob) { 543 if (glob_matches(filename, f->any_glob)) 544 return true; 545 /* If we don't know program name yet, any_glob filter 546 * has to assume that current BPF object file might be 547 * relevant; we'll check again later on after opening 548 * BPF object file, at which point program name will 549 * be known finally. 550 */ 551 if (!prog_name || glob_matches(prog_name, f->any_glob)) 552 return true; 553 } else { 554 if (f->file_glob && !glob_matches(filename, f->file_glob)) 555 continue; 556 if (f->prog_glob && prog_name && !glob_matches(prog_name, f->prog_glob)) 557 continue; 558 return true; 559 } 560 } 561 562 /* if there are no file/prog name allow filters, allow all progs, 563 * unless they are denied earlier explicitly 564 */ 565 return allow_cnt == 0; 566 } 567 568 static struct { 569 enum operator_kind op_kind; 570 const char *op_str; 571 } operators[] = { 572 /* Order of these definitions matter to avoid situations like '<' 573 * matching part of what is actually a '<>' operator. That is, 574 * substrings should go last. 575 */ 576 { OP_EQ, "==" }, 577 { OP_NEQ, "!=" }, 578 { OP_NEQ, "<>" }, 579 { OP_LE, "<=" }, 580 { OP_LT, "<" }, 581 { OP_GE, ">=" }, 582 { OP_GT, ">" }, 583 { OP_EQ, "=" }, 584 }; 585 586 static bool parse_stat_id_var(const char *name, size_t len, int *id, 587 enum stat_variant *var, bool *is_abs); 588 589 static int append_filter(struct filter **filters, int *cnt, const char *str) 590 { 591 struct filter *f; 592 void *tmp; 593 const char *p; 594 int i; 595 596 tmp = realloc(*filters, (*cnt + 1) * sizeof(**filters)); 597 if (!tmp) 598 return -ENOMEM; 599 *filters = tmp; 600 601 f = &(*filters)[*cnt]; 602 memset(f, 0, sizeof(*f)); 603 604 /* First, let's check if it's a stats filter of the following form: 605 * <stat><op><value, where: 606 * - <stat> is one of supported numerical stats (verdict is also 607 * considered numerical, failure == 0, success == 1); 608 * - <op> is comparison operator (see `operators` definitions); 609 * - <value> is an integer (or failure/success, or false/true as 610 * special aliases for 0 and 1, respectively). 611 * If the form doesn't match what user provided, we assume file/prog 612 * glob filter. 613 */ 614 for (i = 0; i < ARRAY_SIZE(operators); i++) { 615 enum stat_variant var; 616 int id; 617 long val; 618 const char *end = str; 619 const char *op_str; 620 bool is_abs; 621 622 op_str = operators[i].op_str; 623 p = strstr(str, op_str); 624 if (!p) 625 continue; 626 627 if (!parse_stat_id_var(str, p - str, &id, &var, &is_abs)) { 628 fprintf(stderr, "Unrecognized stat name in '%s'!\n", str); 629 return -EINVAL; 630 } 631 if (id >= FILE_NAME) { 632 fprintf(stderr, "Non-integer stat is specified in '%s'!\n", str); 633 return -EINVAL; 634 } 635 636 p += strlen(op_str); 637 638 if (strcasecmp(p, "true") == 0 || 639 strcasecmp(p, "t") == 0 || 640 strcasecmp(p, "success") == 0 || 641 strcasecmp(p, "succ") == 0 || 642 strcasecmp(p, "s") == 0 || 643 strcasecmp(p, "match") == 0 || 644 strcasecmp(p, "m") == 0) { 645 val = 1; 646 } else if (strcasecmp(p, "false") == 0 || 647 strcasecmp(p, "f") == 0 || 648 strcasecmp(p, "failure") == 0 || 649 strcasecmp(p, "fail") == 0 || 650 strcasecmp(p, "mismatch") == 0 || 651 strcasecmp(p, "mis") == 0) { 652 val = 0; 653 } else { 654 errno = 0; 655 val = strtol(p, (char **)&end, 10); 656 if (errno || end == p || *end != '\0' ) { 657 fprintf(stderr, "Invalid integer value in '%s'!\n", str); 658 return -EINVAL; 659 } 660 } 661 662 f->kind = FILTER_STAT; 663 f->stat_id = id; 664 f->stat_var = var; 665 f->op = operators[i].op_kind; 666 f->abs = true; 667 f->value = val; 668 669 *cnt += 1; 670 return 0; 671 } 672 673 /* File/prog filter can be specified either as '<glob>' or 674 * '<file-glob>/<prog-glob>'. In the former case <glob> is applied to 675 * both file and program names. This seems to be way more useful in 676 * practice. If user needs full control, they can use '/<prog-glob>' 677 * form to glob just program name, or '<file-glob>/' to glob only file 678 * name. But usually common <glob> seems to be the most useful and 679 * ergonomic way. 680 */ 681 f->kind = FILTER_NAME; 682 p = strchr(str, '/'); 683 if (!p) { 684 f->any_glob = strdup(str); 685 if (!f->any_glob) 686 return -ENOMEM; 687 } else { 688 if (str != p) { 689 /* non-empty file glob */ 690 f->file_glob = strndup(str, p - str); 691 if (!f->file_glob) 692 return -ENOMEM; 693 } 694 if (strlen(p + 1) > 0) { 695 /* non-empty prog glob */ 696 f->prog_glob = strdup(p + 1); 697 if (!f->prog_glob) { 698 free(f->file_glob); 699 f->file_glob = NULL; 700 return -ENOMEM; 701 } 702 } 703 } 704 705 *cnt += 1; 706 return 0; 707 } 708 709 static int append_filter_file(const char *path) 710 { 711 char buf[1024]; 712 FILE *f; 713 int err = 0; 714 715 f = fopen(path, "r"); 716 if (!f) { 717 err = -errno; 718 fprintf(stderr, "Failed to open filters in '%s': %s\n", path, strerror(-err)); 719 return err; 720 } 721 722 while (fscanf(f, " %1023[^\n]\n", buf) == 1) { 723 /* lines starting with # are comments, skip them */ 724 if (buf[0] == '\0' || buf[0] == '#') 725 continue; 726 /* lines starting with ! are negative match filters */ 727 if (buf[0] == '!') 728 err = append_filter(&env.deny_filters, &env.deny_filter_cnt, buf + 1); 729 else 730 err = append_filter(&env.allow_filters, &env.allow_filter_cnt, buf); 731 if (err) 732 goto cleanup; 733 } 734 735 cleanup: 736 fclose(f); 737 return err; 738 } 739 740 static const struct stat_specs default_output_spec = { 741 .spec_cnt = 8, 742 .ids = { 743 FILE_NAME, PROG_NAME, VERDICT, DURATION, 744 TOTAL_INSNS, TOTAL_STATES, SIZE, JITED_SIZE 745 }, 746 }; 747 748 static int append_file(const char *path) 749 { 750 void *tmp; 751 752 tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames)); 753 if (!tmp) 754 return -ENOMEM; 755 env.filenames = tmp; 756 env.filenames[env.filename_cnt] = strdup(path); 757 if (!env.filenames[env.filename_cnt]) 758 return -ENOMEM; 759 env.filename_cnt++; 760 return 0; 761 } 762 763 static int append_file_from_file(const char *path) 764 { 765 char buf[1024]; 766 int err = 0; 767 FILE *f; 768 769 f = fopen(path, "r"); 770 if (!f) { 771 err = -errno; 772 fprintf(stderr, "Failed to open object files list in '%s': %s\n", 773 path, strerror(errno)); 774 return err; 775 } 776 777 while (fscanf(f, " %1023[^\n]\n", buf) == 1) { 778 /* lines starting with # are comments, skip them */ 779 if (buf[0] == '\0' || buf[0] == '#') 780 continue; 781 err = append_file(buf); 782 if (err) 783 goto cleanup; 784 } 785 786 cleanup: 787 fclose(f); 788 return err; 789 } 790 791 static const struct stat_specs default_csv_output_spec = { 792 .spec_cnt = 15, 793 .ids = { 794 FILE_NAME, PROG_NAME, VERDICT, DURATION, 795 TOTAL_INSNS, TOTAL_STATES, PEAK_STATES, 796 MAX_STATES_PER_INSN, MARK_READ_MAX_LEN, 797 SIZE, JITED_SIZE, PROG_TYPE, ATTACH_TYPE, 798 STACK, MEMORY_PEAK, 799 }, 800 }; 801 802 static const struct stat_specs default_sort_spec = { 803 .spec_cnt = 2, 804 .ids = { 805 FILE_NAME, PROG_NAME, 806 }, 807 .asc = { true, true, }, 808 }; 809 810 /* sorting for comparison mode to join two data sets */ 811 static const struct stat_specs join_sort_spec = { 812 .spec_cnt = 2, 813 .ids = { 814 FILE_NAME, PROG_NAME, 815 }, 816 .asc = { true, true, }, 817 }; 818 819 static struct stat_def { 820 const char *header; 821 const char *names[4]; 822 bool asc_by_default; 823 bool left_aligned; 824 } stat_defs[] = { 825 [FILE_NAME] = { "File", {"file_name", "filename", "file"}, true /* asc */, true /* left */ }, 826 [PROG_NAME] = { "Program", {"prog_name", "progname", "prog"}, true /* asc */, true /* left */ }, 827 [VERDICT] = { "Verdict", {"verdict"}, true /* asc: failure, success */, true /* left */ }, 828 [DURATION] = { "Duration (us)", {"duration", "dur"}, }, 829 [TOTAL_INSNS] = { "Insns", {"total_insns", "insns"}, }, 830 [TOTAL_STATES] = { "States", {"total_states", "states"}, }, 831 [PEAK_STATES] = { "Peak states", {"peak_states"}, }, 832 [MAX_STATES_PER_INSN] = { "Max states per insn", {"max_states_per_insn"}, }, 833 [MARK_READ_MAX_LEN] = { "Max mark read length", {"max_mark_read_len", "mark_read"}, }, 834 [SIZE] = { "Program size", {"prog_size"}, }, 835 [JITED_SIZE] = { "Jited size", {"prog_size_jited"}, }, 836 [STACK] = {"Stack depth", {"stack_depth", "stack"}, }, 837 [PROG_TYPE] = { "Program type", {"prog_type"}, }, 838 [ATTACH_TYPE] = { "Attach type", {"attach_type", }, }, 839 [MEMORY_PEAK] = { "Peak memory (MiB)", {"mem_peak", }, }, 840 }; 841 842 static bool parse_stat_id_var(const char *name, size_t len, int *id, 843 enum stat_variant *var, bool *is_abs) 844 { 845 static const char *var_sfxs[] = { 846 [VARIANT_A] = "_a", 847 [VARIANT_B] = "_b", 848 [VARIANT_DIFF] = "_diff", 849 [VARIANT_PCT] = "_pct", 850 }; 851 int i, j, k; 852 853 /* |<stat>| means we take absolute value of given stat */ 854 *is_abs = false; 855 if (len > 2 && name[0] == '|' && name[len - 1] == '|') { 856 *is_abs = true; 857 name += 1; 858 len -= 2; 859 } 860 861 for (i = 0; i < ARRAY_SIZE(stat_defs); i++) { 862 struct stat_def *def = &stat_defs[i]; 863 size_t alias_len, sfx_len; 864 const char *alias; 865 866 for (j = 0; j < ARRAY_SIZE(stat_defs[i].names); j++) { 867 alias = def->names[j]; 868 if (!alias) 869 continue; 870 871 alias_len = strlen(alias); 872 if (strncmp(name, alias, alias_len) != 0) 873 continue; 874 875 if (alias_len == len) { 876 /* If no variant suffix is specified, we 877 * assume control group (just in case we are 878 * in comparison mode. Variant is ignored in 879 * non-comparison mode. 880 */ 881 *var = VARIANT_B; 882 *id = i; 883 return true; 884 } 885 886 for (k = 0; k < ARRAY_SIZE(var_sfxs); k++) { 887 sfx_len = strlen(var_sfxs[k]); 888 if (alias_len + sfx_len != len) 889 continue; 890 891 if (strncmp(name + alias_len, var_sfxs[k], sfx_len) == 0) { 892 *var = (enum stat_variant)k; 893 *id = i; 894 return true; 895 } 896 } 897 } 898 } 899 900 return false; 901 } 902 903 static bool is_asc_sym(char c) 904 { 905 return c == '^'; 906 } 907 908 static bool is_desc_sym(char c) 909 { 910 return c == 'v' || c == 'V' || c == '.' || c == '!' || c == '_'; 911 } 912 913 static char *rtrim(char *str) 914 { 915 int i; 916 917 for (i = strlen(str) - 1; i > 0; --i) { 918 if (!isspace(str[i])) 919 break; 920 str[i] = '\0'; 921 } 922 return str; 923 } 924 925 static int parse_stat(const char *stat_name, struct stat_specs *specs) 926 { 927 int id; 928 bool has_order = false, is_asc = false, is_abs = false; 929 size_t len = strlen(stat_name); 930 enum stat_variant var; 931 932 if (specs->spec_cnt >= ARRAY_SIZE(specs->ids)) { 933 fprintf(stderr, "Can't specify more than %zd stats\n", ARRAY_SIZE(specs->ids)); 934 return -E2BIG; 935 } 936 937 if (len > 1 && (is_asc_sym(stat_name[len - 1]) || is_desc_sym(stat_name[len - 1]))) { 938 has_order = true; 939 is_asc = is_asc_sym(stat_name[len - 1]); 940 len -= 1; 941 } 942 943 if (!parse_stat_id_var(stat_name, len, &id, &var, &is_abs)) { 944 fprintf(stderr, "Unrecognized stat name '%s'\n", stat_name); 945 return -ESRCH; 946 } 947 948 specs->ids[specs->spec_cnt] = id; 949 specs->variants[specs->spec_cnt] = var; 950 specs->asc[specs->spec_cnt] = has_order ? is_asc : stat_defs[id].asc_by_default; 951 specs->abs[specs->spec_cnt] = is_abs; 952 specs->spec_cnt++; 953 954 return 0; 955 } 956 957 static int parse_stats(const char *stats_str, struct stat_specs *specs) 958 { 959 char *input, *state = NULL, *next; 960 int err, cnt = 0; 961 962 input = strdup(stats_str); 963 if (!input) 964 return -ENOMEM; 965 966 while ((next = strtok_r(cnt++ ? NULL : input, ",", &state))) { 967 err = parse_stat(next, specs); 968 if (err) { 969 free(input); 970 return err; 971 } 972 } 973 974 free(input); 975 return 0; 976 } 977 978 static void free_verif_stats(struct verif_stats *stats, size_t stat_cnt) 979 { 980 int i; 981 982 if (!stats) 983 return; 984 985 for (i = 0; i < stat_cnt; i++) { 986 free(stats[i].file_name); 987 free(stats[i].prog_name); 988 } 989 free(stats); 990 } 991 992 static char verif_log_buf[64 * 1024]; 993 994 #define MAX_PARSED_LOG_LINES 100 995 996 static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats *s) 997 { 998 const char *cur; 999 int pos, lines, sub_stack, cnt = 0; 1000 char *state = NULL, *token, stack[512]; 1001 1002 buf[buf_sz - 1] = '\0'; 1003 1004 for (pos = strlen(buf) - 1, lines = 0; pos >= 0 && lines < MAX_PARSED_LOG_LINES; lines++) { 1005 /* find previous endline or otherwise take the start of log buf */ 1006 for (cur = &buf[pos]; cur > buf && cur[0] != '\n'; cur--, pos--) { 1007 } 1008 /* next time start from end of previous line (or pos goes to <0) */ 1009 pos--; 1010 /* if we found endline, point right after endline symbol; 1011 * otherwise, stay at the beginning of log buf 1012 */ 1013 if (cur[0] == '\n') 1014 cur++; 1015 1016 if (1 == sscanf(cur, "verification time %ld usec\n", &s->stats[DURATION])) 1017 continue; 1018 if (5 == sscanf(cur, "processed %ld insns (limit %*d) max_states_per_insn %ld total_states %ld peak_states %ld mark_read %ld", 1019 &s->stats[TOTAL_INSNS], 1020 &s->stats[MAX_STATES_PER_INSN], 1021 &s->stats[TOTAL_STATES], 1022 &s->stats[PEAK_STATES], 1023 &s->stats[MARK_READ_MAX_LEN])) 1024 continue; 1025 1026 if (1 == sscanf(cur, "stack depth %511s", stack)) 1027 continue; 1028 } 1029 while ((token = strtok_r(cnt++ ? NULL : stack, "+", &state))) { 1030 if (sscanf(token, "%d", &sub_stack) == 0) 1031 break; 1032 s->stats[STACK] += sub_stack; 1033 } 1034 return 0; 1035 } 1036 1037 struct line_cnt { 1038 char *line; 1039 int cnt; 1040 }; 1041 1042 static int str_cmp(const void *a, const void *b) 1043 { 1044 const char **str1 = (const char **)a; 1045 const char **str2 = (const char **)b; 1046 1047 return strcmp(*str1, *str2); 1048 } 1049 1050 static int line_cnt_cmp(const void *a, const void *b) 1051 { 1052 const struct line_cnt *a_cnt = (const struct line_cnt *)a; 1053 const struct line_cnt *b_cnt = (const struct line_cnt *)b; 1054 1055 if (a_cnt->cnt != b_cnt->cnt) 1056 return a_cnt->cnt > b_cnt->cnt ? -1 : 1; 1057 return strcmp(a_cnt->line, b_cnt->line); 1058 } 1059 1060 static int print_top_src_lines(char * const buf, size_t buf_sz, const char *prog_name) 1061 { 1062 int lines_cap = 0; 1063 int lines_size = 0; 1064 char **lines = NULL; 1065 char *line = NULL; 1066 char *state; 1067 struct line_cnt *freq = NULL; 1068 struct line_cnt *cur; 1069 int unique_lines; 1070 int err = 0; 1071 int i; 1072 1073 while ((line = strtok_r(line ? NULL : buf, "\n", &state))) { 1074 if (strncmp(line, "; ", 2) != 0) 1075 continue; 1076 line += 2; 1077 1078 if (lines_size == lines_cap) { 1079 char **tmp; 1080 1081 lines_cap = max(16, lines_cap * 2); 1082 tmp = realloc(lines, lines_cap * sizeof(*tmp)); 1083 if (!tmp) { 1084 err = -ENOMEM; 1085 goto cleanup; 1086 } 1087 lines = tmp; 1088 } 1089 lines[lines_size] = line; 1090 lines_size++; 1091 } 1092 1093 if (lines_size == 0) 1094 goto cleanup; 1095 1096 qsort(lines, lines_size, sizeof(*lines), str_cmp); 1097 1098 freq = calloc(lines_size, sizeof(*freq)); 1099 if (!freq) { 1100 err = -ENOMEM; 1101 goto cleanup; 1102 } 1103 1104 cur = freq; 1105 cur->line = lines[0]; 1106 cur->cnt = 1; 1107 for (i = 1; i < lines_size; ++i) { 1108 if (strcmp(lines[i], cur->line) != 0) { 1109 cur++; 1110 cur->line = lines[i]; 1111 cur->cnt = 0; 1112 } 1113 cur->cnt++; 1114 } 1115 unique_lines = cur - freq + 1; 1116 1117 qsort(freq, unique_lines, sizeof(struct line_cnt), line_cnt_cmp); 1118 1119 printf("Top source lines (%s):\n", prog_name); 1120 for (i = 0; i < min(unique_lines, env.top_src_lines); ++i) { 1121 const char *src_code = freq[i].line; 1122 const char *src_line = NULL; 1123 char *split = strrchr(freq[i].line, '@'); 1124 1125 if (split) { 1126 src_line = split + 1; 1127 1128 while (*src_line && isspace(*src_line)) 1129 src_line++; 1130 1131 while (split > src_code && isspace(*split)) 1132 split--; 1133 *split = '\0'; 1134 } 1135 1136 if (src_line) 1137 printf("%5d: (%s)\t%s\n", freq[i].cnt, src_line, src_code); 1138 else 1139 printf("%5d: %s\n", freq[i].cnt, src_code); 1140 } 1141 printf("\n"); 1142 1143 cleanup: 1144 free(freq); 1145 free(lines); 1146 return err; 1147 } 1148 1149 static int guess_prog_type_by_ctx_name(const char *ctx_name, 1150 enum bpf_prog_type *prog_type, 1151 enum bpf_attach_type *attach_type) 1152 { 1153 /* We need to guess program type based on its declared context type. 1154 * This guess can't be perfect as many different program types might 1155 * share the same context type. So we can only hope to reasonably 1156 * well guess this and get lucky. 1157 * 1158 * Just in case, we support both UAPI-side type names and 1159 * kernel-internal names. 1160 */ 1161 static struct { 1162 const char *uapi_name; 1163 const char *kern_name; 1164 enum bpf_prog_type prog_type; 1165 enum bpf_attach_type attach_type; 1166 } ctx_map[] = { 1167 /* __sk_buff is most ambiguous, we assume TC program */ 1168 { "__sk_buff", "sk_buff", BPF_PROG_TYPE_SCHED_CLS }, 1169 { "bpf_sock", "sock", BPF_PROG_TYPE_CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND }, 1170 { "bpf_sock_addr", "bpf_sock_addr_kern", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND }, 1171 { "bpf_sock_ops", "bpf_sock_ops_kern", BPF_PROG_TYPE_SOCK_OPS, BPF_CGROUP_SOCK_OPS }, 1172 { "sk_msg_md", "sk_msg", BPF_PROG_TYPE_SK_MSG, BPF_SK_MSG_VERDICT }, 1173 { "bpf_cgroup_dev_ctx", "bpf_cgroup_dev_ctx", BPF_PROG_TYPE_CGROUP_DEVICE, BPF_CGROUP_DEVICE }, 1174 { "bpf_sysctl", "bpf_sysctl_kern", BPF_PROG_TYPE_CGROUP_SYSCTL, BPF_CGROUP_SYSCTL }, 1175 { "bpf_sockopt", "bpf_sockopt_kern", BPF_PROG_TYPE_CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT }, 1176 { "sk_reuseport_md", "sk_reuseport_kern", BPF_PROG_TYPE_SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE }, 1177 { "bpf_sk_lookup", "bpf_sk_lookup_kern", BPF_PROG_TYPE_SK_LOOKUP, BPF_SK_LOOKUP }, 1178 { "xdp_md", "xdp_buff", BPF_PROG_TYPE_XDP, BPF_XDP }, 1179 /* tracing types with no expected attach type */ 1180 { "bpf_user_pt_regs_t", "pt_regs", BPF_PROG_TYPE_KPROBE }, 1181 { "bpf_perf_event_data", "bpf_perf_event_data_kern", BPF_PROG_TYPE_PERF_EVENT }, 1182 /* raw_tp programs use u64[] from kernel side, we don't want 1183 * to match on that, probably; so NULL for kern-side type 1184 */ 1185 { "bpf_raw_tracepoint_args", NULL, BPF_PROG_TYPE_RAW_TRACEPOINT }, 1186 }; 1187 int i; 1188 1189 if (!ctx_name) 1190 return -EINVAL; 1191 1192 for (i = 0; i < ARRAY_SIZE(ctx_map); i++) { 1193 if (strcmp(ctx_map[i].uapi_name, ctx_name) == 0 || 1194 (ctx_map[i].kern_name && strcmp(ctx_map[i].kern_name, ctx_name) == 0)) { 1195 *prog_type = ctx_map[i].prog_type; 1196 *attach_type = ctx_map[i].attach_type; 1197 return 0; 1198 } 1199 } 1200 1201 return -ESRCH; 1202 } 1203 1204 /* Make sure only target program is referenced from struct_ops map, 1205 * otherwise libbpf would automatically set autocreate for all 1206 * referenced programs. 1207 * See libbpf.c:bpf_object_adjust_struct_ops_autoload. 1208 */ 1209 static void mask_unrelated_struct_ops_progs(struct bpf_object *obj, 1210 struct bpf_map *map, 1211 struct bpf_program *prog) 1212 { 1213 struct btf *btf = bpf_object__btf(obj); 1214 const struct btf_type *t, *mt; 1215 struct btf_member *m; 1216 int i, moff; 1217 size_t data_sz, ptr_sz = sizeof(void *); 1218 void *data; 1219 1220 t = btf__type_by_id(btf, bpf_map__btf_value_type_id(map)); 1221 if (!btf_is_struct(t)) 1222 return; 1223 1224 data = bpf_map__initial_value(map, &data_sz); 1225 for (i = 0; i < btf_vlen(t); i++) { 1226 m = &btf_members(t)[i]; 1227 mt = btf__type_by_id(btf, m->type); 1228 if (!btf_is_ptr(mt)) 1229 continue; 1230 moff = m->offset / 8; 1231 if (moff + ptr_sz > data_sz) 1232 continue; 1233 if (memcmp(data + moff, &prog, ptr_sz) == 0) 1234 continue; 1235 memset(data + moff, 0, ptr_sz); 1236 } 1237 } 1238 1239 static void fixup_obj(struct bpf_object *obj, struct bpf_program *prog, const char *filename) 1240 { 1241 struct bpf_map *map; 1242 1243 bpf_object__for_each_map(map, obj) { 1244 /* disable pinning */ 1245 bpf_map__set_pin_path(map, NULL); 1246 1247 /* fix up map size, if necessary */ 1248 switch (bpf_map__type(map)) { 1249 case BPF_MAP_TYPE_SK_STORAGE: 1250 case BPF_MAP_TYPE_TASK_STORAGE: 1251 case BPF_MAP_TYPE_INODE_STORAGE: 1252 case BPF_MAP_TYPE_CGROUP_STORAGE: 1253 case BPF_MAP_TYPE_CGRP_STORAGE: 1254 break; 1255 case BPF_MAP_TYPE_STRUCT_OPS: 1256 mask_unrelated_struct_ops_progs(obj, map, prog); 1257 break; 1258 default: 1259 if (bpf_map__max_entries(map) == 0) 1260 bpf_map__set_max_entries(map, 1); 1261 } 1262 } 1263 1264 /* SEC(freplace) programs can't be loaded with veristat as is, 1265 * but we can try guessing their target program's expected type by 1266 * looking at the type of program's first argument and substituting 1267 * corresponding program type 1268 */ 1269 if (bpf_program__type(prog) == BPF_PROG_TYPE_EXT) { 1270 const struct btf *btf = bpf_object__btf(obj); 1271 const char *prog_name = bpf_program__name(prog); 1272 enum bpf_prog_type prog_type; 1273 enum bpf_attach_type attach_type; 1274 const struct btf_type *t; 1275 const char *ctx_name; 1276 int id; 1277 1278 if (!btf) 1279 goto skip_freplace_fixup; 1280 1281 id = btf__find_by_name_kind(btf, prog_name, BTF_KIND_FUNC); 1282 t = btf__type_by_id(btf, id); 1283 t = btf__type_by_id(btf, t->type); 1284 if (!btf_is_func_proto(t) || btf_vlen(t) != 1) 1285 goto skip_freplace_fixup; 1286 1287 /* context argument is a pointer to a struct/typedef */ 1288 t = btf__type_by_id(btf, btf_params(t)[0].type); 1289 while (t && btf_is_mod(t)) 1290 t = btf__type_by_id(btf, t->type); 1291 if (!t || !btf_is_ptr(t)) 1292 goto skip_freplace_fixup; 1293 t = btf__type_by_id(btf, t->type); 1294 while (t && btf_is_mod(t)) 1295 t = btf__type_by_id(btf, t->type); 1296 if (!t) 1297 goto skip_freplace_fixup; 1298 1299 ctx_name = btf__name_by_offset(btf, t->name_off); 1300 1301 if (guess_prog_type_by_ctx_name(ctx_name, &prog_type, &attach_type) == 0) { 1302 bpf_program__set_type(prog, prog_type); 1303 bpf_program__set_expected_attach_type(prog, attach_type); 1304 1305 if (!env.quiet) { 1306 fprintf(stderr, "Using guessed program type '%s' for %s/%s...\n", 1307 libbpf_bpf_prog_type_str(prog_type), 1308 filename, prog_name); 1309 } 1310 } else { 1311 if (!env.quiet) { 1312 fprintf(stderr, "Failed to guess program type for freplace program with context type name '%s' for %s/%s. Consider using canonical type names to help veristat...\n", 1313 ctx_name, filename, prog_name); 1314 } 1315 } 1316 } 1317 skip_freplace_fixup: 1318 return; 1319 } 1320 1321 static int max_verifier_log_size(void) 1322 { 1323 const int SMALL_LOG_SIZE = UINT_MAX >> 8; 1324 const int BIG_LOG_SIZE = UINT_MAX >> 2; 1325 struct bpf_insn insns[] = { 1326 { .code = BPF_ALU | BPF_MOV | BPF_X, .dst_reg = BPF_REG_0, }, 1327 { .code = BPF_JMP | BPF_EXIT, }, 1328 }; 1329 LIBBPF_OPTS(bpf_prog_load_opts, opts, 1330 .log_size = BIG_LOG_SIZE, 1331 .log_buf = (void *)-1, 1332 .log_level = 4 1333 ); 1334 int ret, insn_cnt = ARRAY_SIZE(insns); 1335 static int log_size; 1336 1337 if (log_size != 0) 1338 return log_size; 1339 1340 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); 1341 1342 if (ret == -EFAULT) 1343 log_size = BIG_LOG_SIZE; 1344 else /* ret == -EINVAL, big log size is not supported by the verifier */ 1345 log_size = SMALL_LOG_SIZE; 1346 1347 return log_size; 1348 } 1349 1350 static bool output_stat_enabled(int id) 1351 { 1352 int i; 1353 1354 for (i = 0; i < env.output_spec.spec_cnt; i++) 1355 if (env.output_spec.ids[i] == id) 1356 return true; 1357 return false; 1358 } 1359 1360 __attribute__((format(printf, 2, 3))) 1361 static int write_one_line(const char *file, const char *fmt, ...) 1362 { 1363 int err, saved_errno; 1364 va_list ap; 1365 FILE *f; 1366 1367 f = fopen(file, "w"); 1368 if (!f) 1369 return -1; 1370 1371 va_start(ap, fmt); 1372 errno = 0; 1373 err = vfprintf(f, fmt, ap); 1374 saved_errno = errno; 1375 va_end(ap); 1376 fclose(f); 1377 errno = saved_errno; 1378 return err < 0 ? -1 : 0; 1379 } 1380 1381 __attribute__((format(scanf, 3, 4))) 1382 static int scanf_one_line(const char *file, int fields_expected, const char *fmt, ...) 1383 { 1384 int res = 0, saved_errno = 0; 1385 char *line = NULL; 1386 size_t line_len; 1387 va_list ap; 1388 FILE *f; 1389 1390 f = fopen(file, "r"); 1391 if (!f) 1392 return -1; 1393 1394 va_start(ap, fmt); 1395 while (getline(&line, &line_len, f) > 0) { 1396 res = vsscanf(line, fmt, ap); 1397 if (res == fields_expected) 1398 goto out; 1399 } 1400 if (ferror(f)) { 1401 saved_errno = errno; 1402 res = -1; 1403 } 1404 1405 out: 1406 va_end(ap); 1407 free(line); 1408 fclose(f); 1409 errno = saved_errno; 1410 return res; 1411 } 1412 1413 static void destroy_stat_cgroup(void) 1414 { 1415 char buf[PATH_MAX]; 1416 int err; 1417 1418 close(env.memory_peak_fd); 1419 1420 if (env.orig_cgroup[0]) { 1421 snprintf(buf, sizeof(buf), "%s/cgroup.procs", env.orig_cgroup); 1422 err = write_one_line(buf, "%d\n", getpid()); 1423 if (err < 0) 1424 log_errno("moving self to original cgroup %s\n", env.orig_cgroup); 1425 } 1426 1427 if (env.stat_cgroup[0]) { 1428 err = rmdir(env.stat_cgroup); 1429 if (err < 0) 1430 log_errno("deletion of cgroup %s", env.stat_cgroup); 1431 } 1432 1433 env.memory_peak_fd = -1; 1434 env.orig_cgroup[0] = 0; 1435 env.stat_cgroup[0] = 0; 1436 } 1437 1438 /* 1439 * Creates a cgroup at /sys/fs/cgroup/veristat-accounting-<pid>, 1440 * moves current process to this cgroup. 1441 */ 1442 static void create_stat_cgroup(void) 1443 { 1444 char cgroup_fs_mount[4096]; 1445 char buf[4096]; 1446 int err; 1447 1448 env.memory_peak_fd = -1; 1449 1450 if (!output_stat_enabled(MEMORY_PEAK)) 1451 return; 1452 1453 err = scanf_one_line("/proc/self/mounts", 2, "%*s %4095s cgroup2 %s", 1454 cgroup_fs_mount, buf); 1455 if (err != 2) { 1456 if (err < 0) 1457 log_errno("reading /proc/self/mounts"); 1458 else if (!env.quiet) 1459 fprintf(stderr, "Can't find cgroupfs v2 mount point.\n"); 1460 goto err_out; 1461 } 1462 1463 /* cgroup-v2.rst promises the line "0::<group>" for cgroups v2 */ 1464 err = scanf_one_line("/proc/self/cgroup", 1, "0::%4095s", buf); 1465 if (err != 1) { 1466 if (err < 0) 1467 log_errno("reading /proc/self/cgroup"); 1468 else if (!env.quiet) 1469 fprintf(stderr, "Can't infer veristat process cgroup."); 1470 goto err_out; 1471 } 1472 1473 snprintf(env.orig_cgroup, sizeof(env.orig_cgroup), "%s/%s", cgroup_fs_mount, buf); 1474 1475 snprintf(buf, sizeof(buf), "%s/veristat-accounting-%d", cgroup_fs_mount, getpid()); 1476 err = mkdir(buf, 0777); 1477 if (err < 0) { 1478 log_errno("creation of cgroup %s", buf); 1479 goto err_out; 1480 } 1481 strcpy(env.stat_cgroup, buf); 1482 1483 snprintf(buf, sizeof(buf), "%s/cgroup.procs", env.stat_cgroup); 1484 err = write_one_line(buf, "%d\n", getpid()); 1485 if (err < 0) { 1486 log_errno("entering cgroup %s", buf); 1487 goto err_out; 1488 } 1489 1490 snprintf(buf, sizeof(buf), "%s/memory.peak", env.stat_cgroup); 1491 env.memory_peak_fd = open(buf, O_RDWR | O_APPEND); 1492 if (env.memory_peak_fd < 0) { 1493 log_errno("opening %s", buf); 1494 goto err_out; 1495 } 1496 1497 return; 1498 1499 err_out: 1500 if (!env.quiet) 1501 fprintf(stderr, "Memory usage metric unavailable.\n"); 1502 destroy_stat_cgroup(); 1503 } 1504 1505 /* Current value of /sys/fs/cgroup/veristat-accounting-<pid>/memory.peak */ 1506 static long cgroup_memory_peak(void) 1507 { 1508 long err, memory_peak; 1509 char buf[32]; 1510 1511 if (env.memory_peak_fd < 0) 1512 return -1; 1513 1514 err = pread(env.memory_peak_fd, buf, sizeof(buf) - 1, 0); 1515 if (err <= 0) { 1516 log_errno("pread(%s/memory.peak)", env.stat_cgroup); 1517 return -1; 1518 } 1519 1520 buf[err] = 0; 1521 errno = 0; 1522 memory_peak = strtoll(buf, NULL, 10); 1523 if (errno) { 1524 log_errno("%s/memory.peak:strtoll(%s)", env.stat_cgroup, buf); 1525 return -1; 1526 } 1527 1528 return memory_peak; 1529 } 1530 1531 static int reset_stat_cgroup(void) 1532 { 1533 char buf[] = "r\n"; 1534 int err; 1535 1536 if (env.memory_peak_fd < 0) 1537 return -1; 1538 1539 err = pwrite(env.memory_peak_fd, buf, sizeof(buf), 0); 1540 if (err <= 0) { 1541 log_errno("pwrite(%s/memory.peak)", env.stat_cgroup); 1542 return -1; 1543 } 1544 return 0; 1545 } 1546 1547 static int parse_rvalue(const char *val, struct rvalue *rvalue) 1548 { 1549 long long value; 1550 char *val_end; 1551 1552 if (val[0] == '-' || isdigit(val[0])) { 1553 /* must be a number */ 1554 errno = 0; 1555 value = strtoll(val, &val_end, 0); 1556 if (errno == ERANGE) { 1557 errno = 0; 1558 value = strtoull(val, &val_end, 0); 1559 } 1560 if (errno || *val_end != '\0') { 1561 fprintf(stderr, "Failed to parse value '%s'\n", val); 1562 return -EINVAL; 1563 } 1564 rvalue->ivalue = value; 1565 rvalue->type = INTEGRAL; 1566 } else { 1567 /* if not a number, consider it enum value */ 1568 rvalue->svalue = strdup(val); 1569 if (!rvalue->svalue) 1570 return -ENOMEM; 1571 rvalue->type = ENUMERATOR; 1572 } 1573 return 0; 1574 } 1575 1576 static void dump(__u32 prog_id, enum dump_mode mode, const char *file_name, const char *prog_name) 1577 { 1578 char command[64], buf[4096]; 1579 FILE *fp; 1580 int status; 1581 1582 status = system("command -v bpftool > /dev/null 2>&1"); 1583 if (status != 0) { 1584 fprintf(stderr, "bpftool is not available, can't print program dump\n"); 1585 return; 1586 } 1587 snprintf(command, sizeof(command), "bpftool prog dump %s id %u", 1588 mode == DUMP_JITED ? "jited" : "xlated", prog_id); 1589 fp = popen(command, "r"); 1590 if (!fp) { 1591 fprintf(stderr, "bpftool failed with error: %d\n", errno); 1592 return; 1593 } 1594 1595 printf("DUMP (%s) %s/%s:\n", mode == DUMP_JITED ? "JITED" : "XLATED", file_name, prog_name); 1596 while (fgets(buf, sizeof(buf), fp)) 1597 fputs(buf, stdout); 1598 fprintf(stdout, "\n"); 1599 1600 if (ferror(fp)) 1601 fprintf(stderr, "Failed to dump BPF prog with error: %d\n", errno); 1602 1603 pclose(fp); 1604 } 1605 1606 static int process_prog(const char *filename, struct bpf_object *obj, struct bpf_program *prog) 1607 { 1608 const char *base_filename = basename(strdupa(filename)); 1609 const char *prog_name = bpf_program__name(prog); 1610 long mem_peak_a, mem_peak_b, mem_peak = -1; 1611 char *buf; 1612 int buf_sz, log_level; 1613 struct verif_stats *stats; 1614 struct bpf_prog_info info; 1615 __u32 info_len = sizeof(info); 1616 int err = 0, cgroup_err; 1617 void *tmp; 1618 int fd; 1619 1620 if (!should_process_file_prog(base_filename, bpf_program__name(prog))) { 1621 env.progs_skipped++; 1622 return 0; 1623 } 1624 1625 tmp = realloc(env.prog_stats, (env.prog_stat_cnt + 1) * sizeof(*env.prog_stats)); 1626 if (!tmp) 1627 return -ENOMEM; 1628 env.prog_stats = tmp; 1629 stats = &env.prog_stats[env.prog_stat_cnt++]; 1630 memset(stats, 0, sizeof(*stats)); 1631 1632 if (env.verbose || env.top_src_lines > 0) { 1633 buf_sz = env.log_size ? env.log_size : max_verifier_log_size(); 1634 buf = malloc(buf_sz); 1635 if (!buf) 1636 return -ENOMEM; 1637 /* ensure we always request stats */ 1638 log_level = env.log_level | 4 | (env.log_fixed ? 8 : 0); 1639 /* --top-src-lines needs verifier log */ 1640 if (env.top_src_lines > 0 && env.log_level == 0) 1641 log_level |= 2; 1642 } else { 1643 buf = verif_log_buf; 1644 buf_sz = sizeof(verif_log_buf); 1645 /* request only verifier stats */ 1646 log_level = 4 | (env.log_fixed ? 8 : 0); 1647 } 1648 verif_log_buf[0] = '\0'; 1649 1650 bpf_program__set_log_buf(prog, buf, buf_sz); 1651 bpf_program__set_log_level(prog, log_level); 1652 1653 /* increase chances of successful BPF object loading */ 1654 fixup_obj(obj, prog, base_filename); 1655 1656 if (env.force_checkpoints) 1657 bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_STATE_FREQ); 1658 if (env.force_reg_invariants) 1659 bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_REG_INVARIANTS); 1660 1661 err = bpf_object__prepare(obj); 1662 if (!err) { 1663 cgroup_err = reset_stat_cgroup(); 1664 mem_peak_a = cgroup_memory_peak(); 1665 err = bpf_object__load(obj); 1666 mem_peak_b = cgroup_memory_peak(); 1667 if (!cgroup_err && mem_peak_a >= 0 && mem_peak_b >= 0) 1668 mem_peak = mem_peak_b - mem_peak_a; 1669 } 1670 env.progs_processed++; 1671 1672 stats->file_name = strdup(base_filename); 1673 stats->prog_name = strdup(bpf_program__name(prog)); 1674 stats->stats[VERDICT] = err == 0; /* 1 - success, 0 - failure */ 1675 stats->stats[SIZE] = bpf_program__insn_cnt(prog); 1676 stats->stats[PROG_TYPE] = bpf_program__type(prog); 1677 stats->stats[ATTACH_TYPE] = bpf_program__expected_attach_type(prog); 1678 stats->stats[MEMORY_PEAK] = mem_peak < 0 ? -1 : mem_peak / (1024 * 1024); 1679 1680 memset(&info, 0, info_len); 1681 fd = bpf_program__fd(prog); 1682 if (fd > 0 && bpf_prog_get_info_by_fd(fd, &info, &info_len) == 0) { 1683 stats->stats[JITED_SIZE] = info.jited_prog_len; 1684 if (env.dump_mode & DUMP_JITED) 1685 dump(info.id, DUMP_JITED, base_filename, prog_name); 1686 if (env.dump_mode & DUMP_XLATED) 1687 dump(info.id, DUMP_XLATED, base_filename, prog_name); 1688 } 1689 1690 parse_verif_log(buf, buf_sz, stats); 1691 1692 if (env.verbose) { 1693 printf("PROCESSING %s/%s, DURATION US: %ld, VERDICT: %s, VERIFIER LOG:\n%s\n", 1694 filename, prog_name, stats->stats[DURATION], 1695 err ? "failure" : "success", buf); 1696 } 1697 if (env.top_src_lines > 0) 1698 print_top_src_lines(buf, buf_sz, stats->prog_name); 1699 1700 if (verif_log_buf != buf) 1701 free(buf); 1702 1703 return 0; 1704 } 1705 1706 static int append_preset_atom(struct var_preset *preset, char *value, bool is_index) 1707 { 1708 struct field_access *tmp; 1709 int i = preset->atom_count; 1710 int err; 1711 1712 tmp = reallocarray(preset->atoms, i + 1, sizeof(*preset->atoms)); 1713 if (!tmp) 1714 return -ENOMEM; 1715 1716 preset->atoms = tmp; 1717 preset->atom_count++; 1718 1719 if (is_index) { 1720 preset->atoms[i].type = ARRAY_INDEX; 1721 err = parse_rvalue(value, &preset->atoms[i].index); 1722 if (err) 1723 return err; 1724 } else { 1725 preset->atoms[i].type = FIELD_NAME; 1726 preset->atoms[i].name = strdup(value); 1727 if (!preset->atoms[i].name) 1728 return -ENOMEM; 1729 } 1730 return 0; 1731 } 1732 1733 static int parse_var_atoms(const char *full_var, struct var_preset *preset) 1734 { 1735 char expr[256], var[256], *name, *saveptr; 1736 int n, len, off, err; 1737 1738 snprintf(expr, sizeof(expr), "%s", full_var); 1739 preset->atom_count = 0; 1740 while ((name = strtok_r(preset->atom_count ? NULL : expr, ".", &saveptr))) { 1741 len = strlen(name); 1742 /* parse variable name */ 1743 if (sscanf(name, "%[a-zA-Z0-9_] %n", var, &off) != 1) { 1744 fprintf(stderr, "Can't parse %s\n", name); 1745 return -EINVAL; 1746 } 1747 err = append_preset_atom(preset, var, false); 1748 if (err) 1749 return err; 1750 1751 /* parse optional array indexes */ 1752 while (off < len) { 1753 if (sscanf(name + off, " [ %[a-zA-Z0-9_] ] %n", var, &n) != 1) { 1754 fprintf(stderr, "Can't parse %s as index\n", name + off); 1755 return -EINVAL; 1756 } 1757 err = append_preset_atom(preset, var, true); 1758 if (err) 1759 return err; 1760 off += n; 1761 } 1762 } 1763 return 0; 1764 } 1765 1766 static int append_var_preset(struct var_preset **presets, int *cnt, const char *expr) 1767 { 1768 void *tmp; 1769 struct var_preset *cur; 1770 char var[256], val[256]; 1771 int n, err; 1772 1773 tmp = realloc(*presets, (*cnt + 1) * sizeof(**presets)); 1774 if (!tmp) 1775 return -ENOMEM; 1776 *presets = tmp; 1777 cur = &(*presets)[*cnt]; 1778 memset(cur, 0, sizeof(*cur)); 1779 (*cnt)++; 1780 1781 if (sscanf(expr, " %[][a-zA-Z0-9_. ] = %s %n", var, val, &n) != 2 || n != strlen(expr)) { 1782 fprintf(stderr, "Failed to parse expression '%s'\n", expr); 1783 return -EINVAL; 1784 } 1785 /* Remove trailing spaces from var, as scanf may add those */ 1786 rtrim(var); 1787 1788 err = parse_rvalue(val, &cur->value); 1789 if (err) 1790 return err; 1791 1792 cur->full_name = strdup(var); 1793 if (!cur->full_name) 1794 return -ENOMEM; 1795 1796 err = parse_var_atoms(var, cur); 1797 if (err) 1798 return err; 1799 1800 return 0; 1801 } 1802 1803 static int append_var_preset_file(const char *filename) 1804 { 1805 char buf[1024]; 1806 FILE *f; 1807 int err = 0; 1808 1809 f = fopen(filename, "rt"); 1810 if (!f) { 1811 err = -errno; 1812 fprintf(stderr, "Failed to open presets in '%s': %s\n", filename, strerror(-err)); 1813 return -EINVAL; 1814 } 1815 1816 while (fscanf(f, " %1023[^\n]\n", buf) == 1) { 1817 if (buf[0] == '\0' || buf[0] == '#') 1818 continue; 1819 1820 err = append_var_preset(&env.presets, &env.npresets, buf); 1821 if (err) 1822 goto cleanup; 1823 } 1824 1825 cleanup: 1826 fclose(f); 1827 return err; 1828 } 1829 1830 static bool is_signed_type(const struct btf_type *t) 1831 { 1832 if (btf_is_int(t)) 1833 return btf_int_encoding(t) & BTF_INT_SIGNED; 1834 if (btf_is_any_enum(t)) 1835 return btf_kflag(t); 1836 return true; 1837 } 1838 1839 static int enum_value_from_name(const struct btf *btf, const struct btf_type *t, 1840 const char *evalue, long long *retval) 1841 { 1842 if (btf_is_enum(t)) { 1843 struct btf_enum *e = btf_enum(t); 1844 int i, n = btf_vlen(t); 1845 1846 for (i = 0; i < n; ++i, ++e) { 1847 const char *cur_name = btf__name_by_offset(btf, e->name_off); 1848 1849 if (strcmp(cur_name, evalue) == 0) { 1850 *retval = e->val; 1851 return 0; 1852 } 1853 } 1854 } else if (btf_is_enum64(t)) { 1855 struct btf_enum64 *e = btf_enum64(t); 1856 int i, n = btf_vlen(t); 1857 1858 for (i = 0; i < n; ++i, ++e) { 1859 const char *cur_name = btf__name_by_offset(btf, e->name_off); 1860 __u64 value = btf_enum64_value(e); 1861 1862 if (strcmp(cur_name, evalue) == 0) { 1863 *retval = value; 1864 return 0; 1865 } 1866 } 1867 } 1868 return -EINVAL; 1869 } 1870 1871 static bool is_preset_supported(const struct btf_type *t) 1872 { 1873 return btf_is_int(t) || btf_is_enum(t) || btf_is_enum64(t); 1874 } 1875 1876 static int find_enum_value(const struct btf *btf, const char *name, long long *value) 1877 { 1878 const struct btf_type *t; 1879 int cnt, i; 1880 long long lvalue; 1881 1882 cnt = btf__type_cnt(btf); 1883 for (i = 1; i != cnt; ++i) { 1884 t = btf__type_by_id(btf, i); 1885 1886 if (!btf_is_any_enum(t)) 1887 continue; 1888 1889 if (enum_value_from_name(btf, t, name, &lvalue) == 0) { 1890 *value = lvalue; 1891 return 0; 1892 } 1893 } 1894 return -ESRCH; 1895 } 1896 1897 static int resolve_rvalue(struct btf *btf, const struct rvalue *rvalue, long long *result) 1898 { 1899 int err = 0; 1900 1901 switch (rvalue->type) { 1902 case INTEGRAL: 1903 *result = rvalue->ivalue; 1904 return 0; 1905 case ENUMERATOR: 1906 err = find_enum_value(btf, rvalue->svalue, result); 1907 if (err) { 1908 fprintf(stderr, "Can't resolve enum value %s\n", rvalue->svalue); 1909 return err; 1910 } 1911 return 0; 1912 default: 1913 fprintf(stderr, "Unknown rvalue type\n"); 1914 return -EOPNOTSUPP; 1915 } 1916 return 0; 1917 } 1918 1919 static int adjust_var_secinfo_array(struct btf *btf, int tid, struct field_access *atom, 1920 const char *array_name, struct btf_var_secinfo *sinfo) 1921 { 1922 const struct btf_type *t; 1923 struct btf_array *barr; 1924 long long idx; 1925 int err; 1926 1927 tid = btf__resolve_type(btf, tid); 1928 t = btf__type_by_id(btf, tid); 1929 if (!btf_is_array(t)) { 1930 fprintf(stderr, "Array index is not expected for %s\n", 1931 array_name); 1932 return -EINVAL; 1933 } 1934 barr = btf_array(t); 1935 err = resolve_rvalue(btf, &atom->index, &idx); 1936 if (err) 1937 return err; 1938 if (idx < 0 || idx >= barr->nelems) { 1939 fprintf(stderr, "Array index %lld is out of bounds [0, %u): %s\n", 1940 idx, barr->nelems, array_name); 1941 return -EINVAL; 1942 } 1943 sinfo->size = btf__resolve_size(btf, barr->type); 1944 sinfo->offset += sinfo->size * idx; 1945 sinfo->type = btf__resolve_type(btf, barr->type); 1946 return 0; 1947 } 1948 1949 static int adjust_var_secinfo_member(const struct btf *btf, 1950 const struct btf_type *parent_type, 1951 __u32 parent_offset, 1952 const char *member_name, 1953 struct btf_var_secinfo *sinfo) 1954 { 1955 int i; 1956 1957 if (!btf_is_composite(parent_type)) { 1958 fprintf(stderr, "Can't resolve field %s for non-composite type\n", member_name); 1959 return -EINVAL; 1960 } 1961 1962 for (i = 0; i < btf_vlen(parent_type); ++i) { 1963 const struct btf_member *member; 1964 const struct btf_type *member_type; 1965 int tid, off; 1966 1967 member = btf_members(parent_type) + i; 1968 tid = btf__resolve_type(btf, member->type); 1969 if (tid < 0) 1970 return -EINVAL; 1971 1972 member_type = btf__type_by_id(btf, tid); 1973 off = parent_offset + member->offset; 1974 if (member->name_off) { 1975 const char *name = btf__name_by_offset(btf, member->name_off); 1976 1977 if (strcmp(member_name, name) == 0) { 1978 if (btf_member_bitfield_size(parent_type, i) != 0) { 1979 fprintf(stderr, "Bitfield presets are not supported %s\n", 1980 name); 1981 return -EINVAL; 1982 } 1983 sinfo->offset += off / 8; 1984 sinfo->type = tid; 1985 sinfo->size = member_type->size; 1986 return 0; 1987 } 1988 } else if (btf_is_composite(member_type)) { 1989 int err; 1990 1991 err = adjust_var_secinfo_member(btf, member_type, off, 1992 member_name, sinfo); 1993 if (!err) 1994 return 0; 1995 } 1996 } 1997 1998 return -ESRCH; 1999 } 2000 2001 static int adjust_var_secinfo(struct btf *btf, const struct btf_type *t, 2002 struct btf_var_secinfo *sinfo, struct var_preset *preset) 2003 { 2004 const struct btf_type *base_type; 2005 const char *prev_name; 2006 int err, i; 2007 int tid; 2008 2009 assert(preset->atom_count > 0); 2010 assert(preset->atoms[0].type == FIELD_NAME); 2011 2012 tid = btf__resolve_type(btf, t->type); 2013 base_type = btf__type_by_id(btf, tid); 2014 prev_name = preset->atoms[0].name; 2015 2016 for (i = 1; i < preset->atom_count; ++i) { 2017 struct field_access *atom = preset->atoms + i; 2018 2019 switch (atom->type) { 2020 case ARRAY_INDEX: 2021 err = adjust_var_secinfo_array(btf, tid, atom, prev_name, sinfo); 2022 break; 2023 case FIELD_NAME: 2024 err = adjust_var_secinfo_member(btf, base_type, 0, atom->name, sinfo); 2025 if (err == -ESRCH) 2026 fprintf(stderr, "Can't find '%s'\n", atom->name); 2027 prev_name = atom->name; 2028 break; 2029 default: 2030 fprintf(stderr, "Unknown field_access type\n"); 2031 return -EOPNOTSUPP; 2032 } 2033 if (err) 2034 return err; 2035 base_type = btf__type_by_id(btf, sinfo->type); 2036 tid = sinfo->type; 2037 } 2038 2039 return 0; 2040 } 2041 2042 static int set_global_var(struct bpf_object *obj, struct btf *btf, 2043 struct bpf_map *map, struct btf_var_secinfo *sinfo, 2044 struct var_preset *preset) 2045 { 2046 const struct btf_type *base_type; 2047 void *ptr; 2048 long long value = preset->value.ivalue; 2049 size_t size; 2050 2051 base_type = btf__type_by_id(btf, btf__resolve_type(btf, sinfo->type)); 2052 if (!base_type) { 2053 fprintf(stderr, "Failed to resolve type %d\n", sinfo->type); 2054 return -EINVAL; 2055 } 2056 if (!is_preset_supported(base_type)) { 2057 fprintf(stderr, "Can't set %s. Only ints and enums are supported\n", 2058 preset->full_name); 2059 return -EINVAL; 2060 } 2061 2062 if (preset->value.type == ENUMERATOR) { 2063 if (btf_is_any_enum(base_type)) { 2064 if (enum_value_from_name(btf, base_type, preset->value.svalue, &value)) { 2065 fprintf(stderr, 2066 "Failed to find integer value for enum element %s\n", 2067 preset->value.svalue); 2068 return -EINVAL; 2069 } 2070 } else { 2071 fprintf(stderr, "Value %s is not supported for type %s\n", 2072 preset->value.svalue, 2073 btf__name_by_offset(btf, base_type->name_off)); 2074 return -EINVAL; 2075 } 2076 } 2077 2078 /* Check if value fits into the target variable size */ 2079 if (sinfo->size < sizeof(value)) { 2080 bool is_signed = is_signed_type(base_type); 2081 __u32 unsigned_bits = sinfo->size * 8 - (is_signed ? 1 : 0); 2082 long long max_val = 1ll << unsigned_bits; 2083 2084 if (value >= max_val || value < -max_val) { 2085 fprintf(stderr, 2086 "Variable %s value %lld is out of range [%lld; %lld]\n", 2087 btf__name_by_offset(btf, base_type->name_off), value, 2088 is_signed ? -max_val : 0, max_val - 1); 2089 return -EINVAL; 2090 } 2091 } 2092 2093 ptr = bpf_map__initial_value(map, &size); 2094 if (!ptr || sinfo->offset + sinfo->size > size) 2095 return -EINVAL; 2096 2097 if (__BYTE_ORDER == __LITTLE_ENDIAN) { 2098 memcpy(ptr + sinfo->offset, &value, sinfo->size); 2099 } else { /* __BYTE_ORDER == __BIG_ENDIAN */ 2100 __u8 src_offset = sizeof(value) - sinfo->size; 2101 2102 memcpy(ptr + sinfo->offset, (void *)&value + src_offset, sinfo->size); 2103 } 2104 return 0; 2105 } 2106 2107 static int set_global_vars(struct bpf_object *obj, struct var_preset *presets, int npresets) 2108 { 2109 struct btf_var_secinfo *sinfo; 2110 const char *sec_name; 2111 const struct btf_type *t; 2112 struct bpf_map *map; 2113 struct btf *btf; 2114 int i, j, k, n, cnt, err = 0; 2115 2116 if (npresets == 0) 2117 return 0; 2118 2119 btf = bpf_object__btf(obj); 2120 if (!btf) 2121 return -EINVAL; 2122 2123 cnt = btf__type_cnt(btf); 2124 for (i = 1; i != cnt; ++i) { 2125 t = btf__type_by_id(btf, i); 2126 2127 if (!btf_is_datasec(t)) 2128 continue; 2129 2130 sinfo = btf_var_secinfos(t); 2131 sec_name = btf__name_by_offset(btf, t->name_off); 2132 map = bpf_object__find_map_by_name(obj, sec_name); 2133 if (!map) 2134 continue; 2135 2136 n = btf_vlen(t); 2137 for (j = 0; j < n; ++j, ++sinfo) { 2138 const struct btf_type *var_type = btf__type_by_id(btf, sinfo->type); 2139 const char *var_name; 2140 2141 if (!btf_is_var(var_type)) 2142 continue; 2143 2144 var_name = btf__name_by_offset(btf, var_type->name_off); 2145 2146 for (k = 0; k < npresets; ++k) { 2147 struct btf_var_secinfo tmp_sinfo; 2148 2149 if (strcmp(var_name, presets[k].atoms[0].name) != 0) 2150 continue; 2151 2152 if (presets[k].applied) { 2153 fprintf(stderr, "Variable %s is set more than once", 2154 var_name); 2155 return -EINVAL; 2156 } 2157 tmp_sinfo = *sinfo; 2158 err = adjust_var_secinfo(btf, var_type, 2159 &tmp_sinfo, presets + k); 2160 if (err) 2161 return err; 2162 2163 err = set_global_var(obj, btf, map, &tmp_sinfo, presets + k); 2164 if (err) 2165 return err; 2166 2167 presets[k].applied = true; 2168 } 2169 } 2170 } 2171 for (i = 0; i < npresets; ++i) { 2172 if (!presets[i].applied) { 2173 fprintf(stderr, "Global variable preset %s has not been applied\n", 2174 presets[i].full_name); 2175 err = -EINVAL; 2176 } 2177 presets[i].applied = false; 2178 } 2179 return err; 2180 } 2181 2182 static int process_obj(const char *filename) 2183 { 2184 const char *base_filename = basename(strdupa(filename)); 2185 struct bpf_object *obj = NULL, *tobj; 2186 struct bpf_program *prog, *tprog, *lprog; 2187 libbpf_print_fn_t old_libbpf_print_fn; 2188 LIBBPF_OPTS(bpf_object_open_opts, opts); 2189 int err = 0, prog_cnt = 0; 2190 2191 if (!should_process_file_prog(base_filename, NULL)) { 2192 if (env.verbose) 2193 printf("Skipping '%s' due to filters...\n", filename); 2194 env.files_skipped++; 2195 return 0; 2196 } 2197 if (!is_bpf_obj_file(filename)) { 2198 if (env.verbose) 2199 printf("Skipping '%s' as it's not a BPF object file...\n", filename); 2200 env.files_skipped++; 2201 return 0; 2202 } 2203 2204 if (!env.quiet && env.out_fmt == RESFMT_TABLE) 2205 printf("Processing '%s'...\n", base_filename); 2206 2207 old_libbpf_print_fn = libbpf_set_print(libbpf_print_fn); 2208 obj = bpf_object__open_file(filename, &opts); 2209 if (!obj) { 2210 /* if libbpf can't open BPF object file, it could be because 2211 * that BPF object file is incomplete and has to be statically 2212 * linked into a final BPF object file; instead of bailing 2213 * out, report it into stderr, mark it as skipped, and 2214 * proceed 2215 */ 2216 fprintf(stderr, "Failed to open '%s': %d\n", filename, -errno); 2217 env.files_skipped++; 2218 err = 0; 2219 goto cleanup; 2220 } 2221 2222 env.files_processed++; 2223 2224 bpf_object__for_each_program(prog, obj) { 2225 prog_cnt++; 2226 } 2227 2228 if (prog_cnt == 1) { 2229 prog = bpf_object__next_program(obj, NULL); 2230 bpf_program__set_autoload(prog, true); 2231 err = set_global_vars(obj, env.presets, env.npresets); 2232 if (err) { 2233 fprintf(stderr, "Failed to set global variables %d\n", err); 2234 goto cleanup; 2235 } 2236 process_prog(filename, obj, prog); 2237 goto cleanup; 2238 } 2239 2240 bpf_object__for_each_program(prog, obj) { 2241 const char *prog_name = bpf_program__name(prog); 2242 2243 tobj = bpf_object__open_file(filename, &opts); 2244 if (!tobj) { 2245 err = -errno; 2246 fprintf(stderr, "Failed to open '%s': %d\n", filename, err); 2247 goto cleanup; 2248 } 2249 2250 err = set_global_vars(tobj, env.presets, env.npresets); 2251 if (err) { 2252 fprintf(stderr, "Failed to set global variables %d\n", err); 2253 goto cleanup; 2254 } 2255 2256 lprog = NULL; 2257 bpf_object__for_each_program(tprog, tobj) { 2258 const char *tprog_name = bpf_program__name(tprog); 2259 2260 if (strcmp(prog_name, tprog_name) == 0) { 2261 bpf_program__set_autoload(tprog, true); 2262 lprog = tprog; 2263 } else { 2264 bpf_program__set_autoload(tprog, false); 2265 } 2266 } 2267 2268 process_prog(filename, tobj, lprog); 2269 bpf_object__close(tobj); 2270 } 2271 2272 cleanup: 2273 bpf_object__close(obj); 2274 libbpf_set_print(old_libbpf_print_fn); 2275 return err; 2276 } 2277 2278 static int cmp_stat(const struct verif_stats *s1, const struct verif_stats *s2, 2279 enum stat_id id, bool asc, bool abs) 2280 { 2281 int cmp = 0; 2282 2283 switch (id) { 2284 case FILE_NAME: 2285 cmp = strcmp(s1->file_name, s2->file_name); 2286 break; 2287 case PROG_NAME: 2288 cmp = strcmp(s1->prog_name, s2->prog_name); 2289 break; 2290 case ATTACH_TYPE: 2291 case PROG_TYPE: 2292 case SIZE: 2293 case JITED_SIZE: 2294 case STACK: 2295 case VERDICT: 2296 case DURATION: 2297 case TOTAL_INSNS: 2298 case TOTAL_STATES: 2299 case PEAK_STATES: 2300 case MAX_STATES_PER_INSN: 2301 case MEMORY_PEAK: 2302 case MARK_READ_MAX_LEN: { 2303 long v1 = s1->stats[id]; 2304 long v2 = s2->stats[id]; 2305 2306 if (abs) { 2307 v1 = v1 < 0 ? -v1 : v1; 2308 v2 = v2 < 0 ? -v2 : v2; 2309 } 2310 2311 if (v1 != v2) 2312 cmp = v1 < v2 ? -1 : 1; 2313 break; 2314 } 2315 default: 2316 fprintf(stderr, "Unrecognized stat #%d\n", id); 2317 exit(1); 2318 } 2319 2320 return asc ? cmp : -cmp; 2321 } 2322 2323 static int cmp_prog_stats(const void *v1, const void *v2) 2324 { 2325 const struct verif_stats *s1 = v1, *s2 = v2; 2326 int i, cmp; 2327 2328 for (i = 0; i < env.sort_spec.spec_cnt; i++) { 2329 cmp = cmp_stat(s1, s2, env.sort_spec.ids[i], 2330 env.sort_spec.asc[i], env.sort_spec.abs[i]); 2331 if (cmp != 0) 2332 return cmp; 2333 } 2334 2335 /* always disambiguate with file+prog, which are unique */ 2336 cmp = strcmp(s1->file_name, s2->file_name); 2337 if (cmp != 0) 2338 return cmp; 2339 return strcmp(s1->prog_name, s2->prog_name); 2340 } 2341 2342 static void fetch_join_stat_value(const struct verif_stats_join *s, 2343 enum stat_id id, enum stat_variant var, 2344 const char **str_val, 2345 double *num_val) 2346 { 2347 long v1, v2; 2348 2349 if (id == FILE_NAME) { 2350 *str_val = s->file_name; 2351 return; 2352 } 2353 if (id == PROG_NAME) { 2354 *str_val = s->prog_name; 2355 return; 2356 } 2357 2358 v1 = s->stats_a ? s->stats_a->stats[id] : 0; 2359 v2 = s->stats_b ? s->stats_b->stats[id] : 0; 2360 2361 switch (var) { 2362 case VARIANT_A: 2363 if (!s->stats_a) 2364 *num_val = -DBL_MAX; 2365 else 2366 *num_val = s->stats_a->stats[id]; 2367 return; 2368 case VARIANT_B: 2369 if (!s->stats_b) 2370 *num_val = -DBL_MAX; 2371 else 2372 *num_val = s->stats_b->stats[id]; 2373 return; 2374 case VARIANT_DIFF: 2375 if (!s->stats_a || !s->stats_b) 2376 *num_val = -DBL_MAX; 2377 else if (id == VERDICT) 2378 *num_val = v1 == v2 ? 1.0 /* MATCH */ : 0.0 /* MISMATCH */; 2379 else 2380 *num_val = (double)(v2 - v1); 2381 return; 2382 case VARIANT_PCT: 2383 if (!s->stats_a || !s->stats_b) { 2384 *num_val = -DBL_MAX; 2385 } else if (v1 == 0) { 2386 if (v1 == v2) 2387 *num_val = 0.0; 2388 else 2389 *num_val = v2 < v1 ? -100.0 : 100.0; 2390 } else { 2391 *num_val = (v2 - v1) * 100.0 / v1; 2392 } 2393 return; 2394 } 2395 } 2396 2397 static int cmp_join_stat(const struct verif_stats_join *s1, 2398 const struct verif_stats_join *s2, 2399 enum stat_id id, enum stat_variant var, 2400 bool asc, bool abs) 2401 { 2402 const char *str1 = NULL, *str2 = NULL; 2403 double v1 = 0.0, v2 = 0.0; 2404 int cmp = 0; 2405 2406 fetch_join_stat_value(s1, id, var, &str1, &v1); 2407 fetch_join_stat_value(s2, id, var, &str2, &v2); 2408 2409 if (abs) { 2410 v1 = fabs(v1); 2411 v2 = fabs(v2); 2412 } 2413 2414 if (str1) 2415 cmp = strcmp(str1, str2); 2416 else if (v1 != v2) 2417 cmp = v1 < v2 ? -1 : 1; 2418 2419 return asc ? cmp : -cmp; 2420 } 2421 2422 static int cmp_join_stats(const void *v1, const void *v2) 2423 { 2424 const struct verif_stats_join *s1 = v1, *s2 = v2; 2425 int i, cmp; 2426 2427 for (i = 0; i < env.sort_spec.spec_cnt; i++) { 2428 cmp = cmp_join_stat(s1, s2, 2429 env.sort_spec.ids[i], 2430 env.sort_spec.variants[i], 2431 env.sort_spec.asc[i], 2432 env.sort_spec.abs[i]); 2433 if (cmp != 0) 2434 return cmp; 2435 } 2436 2437 /* always disambiguate with file+prog, which are unique */ 2438 cmp = strcmp(s1->file_name, s2->file_name); 2439 if (cmp != 0) 2440 return cmp; 2441 return strcmp(s1->prog_name, s2->prog_name); 2442 } 2443 2444 #define HEADER_CHAR '-' 2445 #define COLUMN_SEP " " 2446 2447 static void output_header_underlines(void) 2448 { 2449 int i, j, len; 2450 2451 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2452 len = env.output_spec.lens[i]; 2453 2454 printf("%s", i == 0 ? "" : COLUMN_SEP); 2455 for (j = 0; j < len; j++) 2456 printf("%c", HEADER_CHAR); 2457 } 2458 printf("\n"); 2459 } 2460 2461 static void output_headers(enum resfmt fmt) 2462 { 2463 const char *fmt_str; 2464 int i, len; 2465 2466 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2467 int id = env.output_spec.ids[i]; 2468 int *max_len = &env.output_spec.lens[i]; 2469 2470 switch (fmt) { 2471 case RESFMT_TABLE_CALCLEN: 2472 len = snprintf(NULL, 0, "%s", stat_defs[id].header); 2473 if (len > *max_len) 2474 *max_len = len; 2475 break; 2476 case RESFMT_TABLE: 2477 fmt_str = stat_defs[id].left_aligned ? "%s%-*s" : "%s%*s"; 2478 printf(fmt_str, i == 0 ? "" : COLUMN_SEP, *max_len, stat_defs[id].header); 2479 if (i == env.output_spec.spec_cnt - 1) 2480 printf("\n"); 2481 break; 2482 case RESFMT_CSV: 2483 printf("%s%s", i == 0 ? "" : ",", stat_defs[id].names[0]); 2484 if (i == env.output_spec.spec_cnt - 1) 2485 printf("\n"); 2486 break; 2487 } 2488 } 2489 2490 if (fmt == RESFMT_TABLE) 2491 output_header_underlines(); 2492 } 2493 2494 static void prepare_value(const struct verif_stats *s, enum stat_id id, 2495 const char **str, long *val) 2496 { 2497 switch (id) { 2498 case FILE_NAME: 2499 *str = s ? s->file_name : "N/A"; 2500 break; 2501 case PROG_NAME: 2502 *str = s ? s->prog_name : "N/A"; 2503 break; 2504 case VERDICT: 2505 if (!s) 2506 *str = "N/A"; 2507 else 2508 *str = s->stats[VERDICT] ? "success" : "failure"; 2509 break; 2510 case ATTACH_TYPE: 2511 if (!s) 2512 *str = "N/A"; 2513 else 2514 *str = libbpf_bpf_attach_type_str(s->stats[ATTACH_TYPE]) ?: "N/A"; 2515 break; 2516 case PROG_TYPE: 2517 if (!s) 2518 *str = "N/A"; 2519 else 2520 *str = libbpf_bpf_prog_type_str(s->stats[PROG_TYPE]) ?: "N/A"; 2521 break; 2522 case DURATION: 2523 case TOTAL_INSNS: 2524 case TOTAL_STATES: 2525 case PEAK_STATES: 2526 case MAX_STATES_PER_INSN: 2527 case MARK_READ_MAX_LEN: 2528 case STACK: 2529 case SIZE: 2530 case JITED_SIZE: 2531 case MEMORY_PEAK: 2532 *val = s ? s->stats[id] : 0; 2533 break; 2534 default: 2535 fprintf(stderr, "Unrecognized stat #%d\n", id); 2536 exit(1); 2537 } 2538 } 2539 2540 static void output_stats(const struct verif_stats *s, enum resfmt fmt, bool last) 2541 { 2542 int i; 2543 2544 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2545 int id = env.output_spec.ids[i]; 2546 int *max_len = &env.output_spec.lens[i], len; 2547 const char *str = NULL; 2548 long val = 0; 2549 2550 prepare_value(s, id, &str, &val); 2551 2552 switch (fmt) { 2553 case RESFMT_TABLE_CALCLEN: 2554 if (str) 2555 len = snprintf(NULL, 0, "%s", str); 2556 else 2557 len = snprintf(NULL, 0, "%ld", val); 2558 if (len > *max_len) 2559 *max_len = len; 2560 break; 2561 case RESFMT_TABLE: 2562 if (str) 2563 printf("%s%-*s", i == 0 ? "" : COLUMN_SEP, *max_len, str); 2564 else 2565 printf("%s%*ld", i == 0 ? "" : COLUMN_SEP, *max_len, val); 2566 if (i == env.output_spec.spec_cnt - 1) 2567 printf("\n"); 2568 break; 2569 case RESFMT_CSV: 2570 if (str) 2571 printf("%s%s", i == 0 ? "" : ",", str); 2572 else 2573 printf("%s%ld", i == 0 ? "" : ",", val); 2574 if (i == env.output_spec.spec_cnt - 1) 2575 printf("\n"); 2576 break; 2577 } 2578 } 2579 2580 if (last && fmt == RESFMT_TABLE) { 2581 output_header_underlines(); 2582 printf("Done. Processed %d files, %d programs. Skipped %d files, %d programs.\n", 2583 env.files_processed, env.files_skipped, env.progs_processed, env.progs_skipped); 2584 } 2585 } 2586 2587 static int parse_stat_value(const char *str, enum stat_id id, struct verif_stats *st) 2588 { 2589 switch (id) { 2590 case FILE_NAME: 2591 st->file_name = strdup(str); 2592 if (!st->file_name) 2593 return -ENOMEM; 2594 break; 2595 case PROG_NAME: 2596 st->prog_name = strdup(str); 2597 if (!st->prog_name) 2598 return -ENOMEM; 2599 break; 2600 case VERDICT: 2601 if (strcmp(str, "success") == 0) { 2602 st->stats[VERDICT] = true; 2603 } else if (strcmp(str, "failure") == 0) { 2604 st->stats[VERDICT] = false; 2605 } else { 2606 fprintf(stderr, "Unrecognized verification verdict '%s'\n", str); 2607 return -EINVAL; 2608 } 2609 break; 2610 case DURATION: 2611 case TOTAL_INSNS: 2612 case TOTAL_STATES: 2613 case PEAK_STATES: 2614 case MAX_STATES_PER_INSN: 2615 case MARK_READ_MAX_LEN: 2616 case SIZE: 2617 case JITED_SIZE: 2618 case MEMORY_PEAK: 2619 case STACK: { 2620 long val; 2621 int err, n; 2622 2623 if (sscanf(str, "%ld %n", &val, &n) != 1 || n != strlen(str)) { 2624 err = -errno; 2625 fprintf(stderr, "Failed to parse '%s' as integer\n", str); 2626 return err; 2627 } 2628 2629 st->stats[id] = val; 2630 break; 2631 } 2632 case PROG_TYPE: { 2633 enum bpf_prog_type prog_type = 0; 2634 const char *type; 2635 2636 while ((type = libbpf_bpf_prog_type_str(prog_type))) { 2637 if (strcmp(type, str) == 0) { 2638 st->stats[id] = prog_type; 2639 break; 2640 } 2641 prog_type++; 2642 } 2643 2644 if (!type) { 2645 fprintf(stderr, "Unrecognized prog type %s\n", str); 2646 return -EINVAL; 2647 } 2648 break; 2649 } 2650 case ATTACH_TYPE: { 2651 enum bpf_attach_type attach_type = 0; 2652 const char *type; 2653 2654 while ((type = libbpf_bpf_attach_type_str(attach_type))) { 2655 if (strcmp(type, str) == 0) { 2656 st->stats[id] = attach_type; 2657 break; 2658 } 2659 attach_type++; 2660 } 2661 2662 if (!type) { 2663 fprintf(stderr, "Unrecognized attach type %s\n", str); 2664 return -EINVAL; 2665 } 2666 break; 2667 } 2668 default: 2669 fprintf(stderr, "Unrecognized stat #%d\n", id); 2670 return -EINVAL; 2671 } 2672 return 0; 2673 } 2674 2675 static int parse_stats_csv(const char *filename, struct stat_specs *specs, 2676 struct verif_stats **statsp, int *stat_cntp) 2677 { 2678 char line[4096]; 2679 FILE *f; 2680 int err = 0; 2681 bool header = true; 2682 2683 f = fopen(filename, "r"); 2684 if (!f) { 2685 err = -errno; 2686 fprintf(stderr, "Failed to open '%s': %d\n", filename, err); 2687 return err; 2688 } 2689 2690 *stat_cntp = 0; 2691 2692 while (fgets(line, sizeof(line), f)) { 2693 char *input = line, *state = NULL, *next; 2694 struct verif_stats *st = NULL; 2695 int col = 0, cnt = 0; 2696 2697 if (!header) { 2698 void *tmp; 2699 2700 tmp = realloc(*statsp, (*stat_cntp + 1) * sizeof(**statsp)); 2701 if (!tmp) { 2702 err = -ENOMEM; 2703 goto cleanup; 2704 } 2705 *statsp = tmp; 2706 2707 st = &(*statsp)[*stat_cntp]; 2708 memset(st, 0, sizeof(*st)); 2709 2710 *stat_cntp += 1; 2711 } 2712 2713 while ((next = strtok_r(cnt++ ? NULL : input, ",\n", &state))) { 2714 if (header) { 2715 /* for the first line, set up spec stats */ 2716 err = parse_stat(next, specs); 2717 if (err) 2718 goto cleanup; 2719 continue; 2720 } 2721 2722 /* for all other lines, parse values based on spec */ 2723 if (col >= specs->spec_cnt) { 2724 fprintf(stderr, "Found extraneous column #%d in row #%d of '%s'\n", 2725 col, *stat_cntp, filename); 2726 err = -EINVAL; 2727 goto cleanup; 2728 } 2729 err = parse_stat_value(next, specs->ids[col], st); 2730 if (err) 2731 goto cleanup; 2732 col++; 2733 } 2734 2735 if (header) { 2736 header = false; 2737 continue; 2738 } 2739 2740 if (col < specs->spec_cnt) { 2741 fprintf(stderr, "Not enough columns in row #%d in '%s'\n", 2742 *stat_cntp, filename); 2743 err = -EINVAL; 2744 goto cleanup; 2745 } 2746 2747 if (!st->file_name || !st->prog_name) { 2748 fprintf(stderr, "Row #%d in '%s' is missing file and/or program name\n", 2749 *stat_cntp, filename); 2750 err = -EINVAL; 2751 goto cleanup; 2752 } 2753 2754 /* in comparison mode we can only check filters after we 2755 * parsed entire line; if row should be ignored we pretend we 2756 * never parsed it 2757 */ 2758 if (!should_process_file_prog(st->file_name, st->prog_name)) { 2759 free(st->file_name); 2760 free(st->prog_name); 2761 *stat_cntp -= 1; 2762 } 2763 } 2764 2765 if (!feof(f)) { 2766 err = -errno; 2767 fprintf(stderr, "Failed I/O for '%s': %d\n", filename, err); 2768 } 2769 2770 cleanup: 2771 fclose(f); 2772 return err; 2773 } 2774 2775 /* empty/zero stats for mismatched rows */ 2776 static const struct verif_stats fallback_stats = { .file_name = "", .prog_name = "" }; 2777 2778 static bool is_key_stat(enum stat_id id) 2779 { 2780 return id == FILE_NAME || id == PROG_NAME; 2781 } 2782 2783 static void output_comp_header_underlines(void) 2784 { 2785 int i, j, k; 2786 2787 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2788 int id = env.output_spec.ids[i]; 2789 int max_j = is_key_stat(id) ? 1 : 3; 2790 2791 for (j = 0; j < max_j; j++) { 2792 int len = env.output_spec.lens[3 * i + j]; 2793 2794 printf("%s", i + j == 0 ? "" : COLUMN_SEP); 2795 2796 for (k = 0; k < len; k++) 2797 printf("%c", HEADER_CHAR); 2798 } 2799 } 2800 printf("\n"); 2801 } 2802 2803 static void output_comp_headers(enum resfmt fmt) 2804 { 2805 static const char *table_sfxs[3] = {" (A)", " (B)", " (DIFF)"}; 2806 static const char *name_sfxs[3] = {"_base", "_comp", "_diff"}; 2807 int i, j, len; 2808 2809 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2810 int id = env.output_spec.ids[i]; 2811 /* key stats don't have A/B/DIFF columns, they are common for both data sets */ 2812 int max_j = is_key_stat(id) ? 1 : 3; 2813 2814 for (j = 0; j < max_j; j++) { 2815 int *max_len = &env.output_spec.lens[3 * i + j]; 2816 bool last = (i == env.output_spec.spec_cnt - 1) && (j == max_j - 1); 2817 const char *sfx; 2818 2819 switch (fmt) { 2820 case RESFMT_TABLE_CALCLEN: 2821 sfx = is_key_stat(id) ? "" : table_sfxs[j]; 2822 len = snprintf(NULL, 0, "%s%s", stat_defs[id].header, sfx); 2823 if (len > *max_len) 2824 *max_len = len; 2825 break; 2826 case RESFMT_TABLE: 2827 sfx = is_key_stat(id) ? "" : table_sfxs[j]; 2828 printf("%s%-*s%s", i + j == 0 ? "" : COLUMN_SEP, 2829 *max_len - (int)strlen(sfx), stat_defs[id].header, sfx); 2830 if (last) 2831 printf("\n"); 2832 break; 2833 case RESFMT_CSV: 2834 sfx = is_key_stat(id) ? "" : name_sfxs[j]; 2835 printf("%s%s%s", i + j == 0 ? "" : ",", stat_defs[id].names[0], sfx); 2836 if (last) 2837 printf("\n"); 2838 break; 2839 } 2840 } 2841 } 2842 2843 if (fmt == RESFMT_TABLE) 2844 output_comp_header_underlines(); 2845 } 2846 2847 static void output_comp_stats(const struct verif_stats_join *join_stats, 2848 enum resfmt fmt, bool last) 2849 { 2850 const struct verif_stats *base = join_stats->stats_a; 2851 const struct verif_stats *comp = join_stats->stats_b; 2852 char base_buf[1024] = {}, comp_buf[1024] = {}, diff_buf[1024] = {}; 2853 int i; 2854 2855 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2856 int id = env.output_spec.ids[i], len; 2857 int *max_len_base = &env.output_spec.lens[3 * i + 0]; 2858 int *max_len_comp = &env.output_spec.lens[3 * i + 1]; 2859 int *max_len_diff = &env.output_spec.lens[3 * i + 2]; 2860 const char *base_str = NULL, *comp_str = NULL; 2861 long base_val = 0, comp_val = 0, diff_val = 0; 2862 2863 prepare_value(base, id, &base_str, &base_val); 2864 prepare_value(comp, id, &comp_str, &comp_val); 2865 2866 /* normalize all the outputs to be in string buffers for simplicity */ 2867 if (is_key_stat(id)) { 2868 /* key stats (file and program name) are always strings */ 2869 if (base) 2870 snprintf(base_buf, sizeof(base_buf), "%s", base_str); 2871 else 2872 snprintf(base_buf, sizeof(base_buf), "%s", comp_str); 2873 } else if (base_str) { 2874 snprintf(base_buf, sizeof(base_buf), "%s", base_str); 2875 snprintf(comp_buf, sizeof(comp_buf), "%s", comp_str); 2876 if (!base || !comp) 2877 snprintf(diff_buf, sizeof(diff_buf), "%s", "N/A"); 2878 else if (strcmp(base_str, comp_str) == 0) 2879 snprintf(diff_buf, sizeof(diff_buf), "%s", "MATCH"); 2880 else 2881 snprintf(diff_buf, sizeof(diff_buf), "%s", "MISMATCH"); 2882 } else { 2883 double p = 0.0; 2884 2885 if (base) 2886 snprintf(base_buf, sizeof(base_buf), "%ld", base_val); 2887 else 2888 snprintf(base_buf, sizeof(base_buf), "%s", "N/A"); 2889 if (comp) 2890 snprintf(comp_buf, sizeof(comp_buf), "%ld", comp_val); 2891 else 2892 snprintf(comp_buf, sizeof(comp_buf), "%s", "N/A"); 2893 2894 diff_val = comp_val - base_val; 2895 if (!base || !comp) { 2896 snprintf(diff_buf, sizeof(diff_buf), "%s", "N/A"); 2897 } else { 2898 if (base_val == 0) { 2899 if (comp_val == base_val) 2900 p = 0.0; /* avoid +0 (+100%) case */ 2901 else 2902 p = comp_val < base_val ? -100.0 : 100.0; 2903 } else { 2904 p = diff_val * 100.0 / base_val; 2905 } 2906 snprintf(diff_buf, sizeof(diff_buf), "%+ld (%+.2lf%%)", diff_val, p); 2907 } 2908 } 2909 2910 switch (fmt) { 2911 case RESFMT_TABLE_CALCLEN: 2912 len = strlen(base_buf); 2913 if (len > *max_len_base) 2914 *max_len_base = len; 2915 if (!is_key_stat(id)) { 2916 len = strlen(comp_buf); 2917 if (len > *max_len_comp) 2918 *max_len_comp = len; 2919 len = strlen(diff_buf); 2920 if (len > *max_len_diff) 2921 *max_len_diff = len; 2922 } 2923 break; 2924 case RESFMT_TABLE: { 2925 /* string outputs are left-aligned, number outputs are right-aligned */ 2926 const char *fmt = base_str ? "%s%-*s" : "%s%*s"; 2927 2928 printf(fmt, i == 0 ? "" : COLUMN_SEP, *max_len_base, base_buf); 2929 if (!is_key_stat(id)) { 2930 printf(fmt, COLUMN_SEP, *max_len_comp, comp_buf); 2931 printf(fmt, COLUMN_SEP, *max_len_diff, diff_buf); 2932 } 2933 if (i == env.output_spec.spec_cnt - 1) 2934 printf("\n"); 2935 break; 2936 } 2937 case RESFMT_CSV: 2938 printf("%s%s", i == 0 ? "" : ",", base_buf); 2939 if (!is_key_stat(id)) { 2940 printf("%s%s", i == 0 ? "" : ",", comp_buf); 2941 printf("%s%s", i == 0 ? "" : ",", diff_buf); 2942 } 2943 if (i == env.output_spec.spec_cnt - 1) 2944 printf("\n"); 2945 break; 2946 } 2947 } 2948 2949 if (last && fmt == RESFMT_TABLE) 2950 output_comp_header_underlines(); 2951 } 2952 2953 static int cmp_stats_key(const struct verif_stats *base, const struct verif_stats *comp) 2954 { 2955 int r; 2956 2957 r = strcmp(base->file_name, comp->file_name); 2958 if (r != 0) 2959 return r; 2960 return strcmp(base->prog_name, comp->prog_name); 2961 } 2962 2963 static bool is_join_stat_filter_matched(struct filter *f, const struct verif_stats_join *stats) 2964 { 2965 static const double eps = 1e-9; 2966 const char *str = NULL; 2967 double value = 0.0; 2968 2969 fetch_join_stat_value(stats, f->stat_id, f->stat_var, &str, &value); 2970 2971 if (f->abs) 2972 value = fabs(value); 2973 2974 switch (f->op) { 2975 case OP_EQ: return value > f->value - eps && value < f->value + eps; 2976 case OP_NEQ: return value < f->value - eps || value > f->value + eps; 2977 case OP_LT: return value < f->value - eps; 2978 case OP_LE: return value <= f->value + eps; 2979 case OP_GT: return value > f->value + eps; 2980 case OP_GE: return value >= f->value - eps; 2981 } 2982 2983 fprintf(stderr, "BUG: unknown filter op %d!\n", f->op); 2984 return false; 2985 } 2986 2987 static bool should_output_join_stats(const struct verif_stats_join *stats) 2988 { 2989 struct filter *f; 2990 int i, allow_cnt = 0; 2991 2992 for (i = 0; i < env.deny_filter_cnt; i++) { 2993 f = &env.deny_filters[i]; 2994 if (f->kind != FILTER_STAT) 2995 continue; 2996 2997 if (is_join_stat_filter_matched(f, stats)) 2998 return false; 2999 } 3000 3001 for (i = 0; i < env.allow_filter_cnt; i++) { 3002 f = &env.allow_filters[i]; 3003 if (f->kind != FILTER_STAT) 3004 continue; 3005 allow_cnt++; 3006 3007 if (is_join_stat_filter_matched(f, stats)) 3008 return true; 3009 } 3010 3011 /* if there are no stat allowed filters, pass everything through */ 3012 return allow_cnt == 0; 3013 } 3014 3015 static int handle_comparison_mode(void) 3016 { 3017 struct stat_specs base_specs = {}, comp_specs = {}; 3018 struct stat_specs tmp_sort_spec; 3019 enum resfmt cur_fmt; 3020 int err, i, j, last_idx, cnt; 3021 3022 if (env.filename_cnt != 2) { 3023 fprintf(stderr, "Comparison mode expects exactly two input CSV files!\n\n"); 3024 argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); 3025 return -EINVAL; 3026 } 3027 3028 err = parse_stats_csv(env.filenames[0], &base_specs, 3029 &env.baseline_stats, &env.baseline_stat_cnt); 3030 if (err) { 3031 fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err); 3032 return err; 3033 } 3034 err = parse_stats_csv(env.filenames[1], &comp_specs, 3035 &env.prog_stats, &env.prog_stat_cnt); 3036 if (err) { 3037 fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[1], err); 3038 return err; 3039 } 3040 3041 /* To keep it simple we validate that the set and order of stats in 3042 * both CSVs are exactly the same. This can be lifted with a bit more 3043 * pre-processing later. 3044 */ 3045 if (base_specs.spec_cnt != comp_specs.spec_cnt) { 3046 fprintf(stderr, "Number of stats in '%s' and '%s' differs (%d != %d)!\n", 3047 env.filenames[0], env.filenames[1], 3048 base_specs.spec_cnt, comp_specs.spec_cnt); 3049 return -EINVAL; 3050 } 3051 for (i = 0; i < base_specs.spec_cnt; i++) { 3052 if (base_specs.ids[i] != comp_specs.ids[i]) { 3053 fprintf(stderr, "Stats composition differs between '%s' and '%s' (%s != %s)!\n", 3054 env.filenames[0], env.filenames[1], 3055 stat_defs[base_specs.ids[i]].names[0], 3056 stat_defs[comp_specs.ids[i]].names[0]); 3057 return -EINVAL; 3058 } 3059 } 3060 3061 /* Replace user-specified sorting spec with file+prog sorting rule to 3062 * be able to join two datasets correctly. Once we are done, we will 3063 * restore the original sort spec. 3064 */ 3065 tmp_sort_spec = env.sort_spec; 3066 env.sort_spec = join_sort_spec; 3067 qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats); 3068 qsort(env.baseline_stats, env.baseline_stat_cnt, sizeof(*env.baseline_stats), cmp_prog_stats); 3069 env.sort_spec = tmp_sort_spec; 3070 3071 /* Join two datasets together. If baseline and comparison datasets 3072 * have different subset of rows (we match by 'object + prog' as 3073 * a unique key) then assume empty/missing/zero value for rows that 3074 * are missing in the opposite data set. 3075 */ 3076 i = j = 0; 3077 while (i < env.baseline_stat_cnt || j < env.prog_stat_cnt) { 3078 const struct verif_stats *base, *comp; 3079 struct verif_stats_join *join; 3080 void *tmp; 3081 int r; 3082 3083 base = i < env.baseline_stat_cnt ? &env.baseline_stats[i] : &fallback_stats; 3084 comp = j < env.prog_stat_cnt ? &env.prog_stats[j] : &fallback_stats; 3085 3086 if (!base->file_name || !base->prog_name) { 3087 fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n", 3088 i, env.filenames[0]); 3089 return -EINVAL; 3090 } 3091 if (!comp->file_name || !comp->prog_name) { 3092 fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n", 3093 j, env.filenames[1]); 3094 return -EINVAL; 3095 } 3096 3097 tmp = realloc(env.join_stats, (env.join_stat_cnt + 1) * sizeof(*env.join_stats)); 3098 if (!tmp) 3099 return -ENOMEM; 3100 env.join_stats = tmp; 3101 3102 join = &env.join_stats[env.join_stat_cnt]; 3103 memset(join, 0, sizeof(*join)); 3104 3105 r = cmp_stats_key(base, comp); 3106 if (r == 0) { 3107 join->file_name = base->file_name; 3108 join->prog_name = base->prog_name; 3109 join->stats_a = base; 3110 join->stats_b = comp; 3111 i++; 3112 j++; 3113 } else if (base != &fallback_stats && (comp == &fallback_stats || r < 0)) { 3114 join->file_name = base->file_name; 3115 join->prog_name = base->prog_name; 3116 join->stats_a = base; 3117 join->stats_b = NULL; 3118 i++; 3119 } else if (comp != &fallback_stats && (base == &fallback_stats || r > 0)) { 3120 join->file_name = comp->file_name; 3121 join->prog_name = comp->prog_name; 3122 join->stats_a = NULL; 3123 join->stats_b = comp; 3124 j++; 3125 } else { 3126 fprintf(stderr, "%s:%d: should never reach here i=%i, j=%i", 3127 __FILE__, __LINE__, i, j); 3128 return -EINVAL; 3129 } 3130 env.join_stat_cnt += 1; 3131 } 3132 3133 /* now sort joined results according to sort spec */ 3134 qsort(env.join_stats, env.join_stat_cnt, sizeof(*env.join_stats), cmp_join_stats); 3135 3136 /* for human-readable table output we need to do extra pass to 3137 * calculate column widths, so we substitute current output format 3138 * with RESFMT_TABLE_CALCLEN and later revert it back to RESFMT_TABLE 3139 * and do everything again. 3140 */ 3141 if (env.out_fmt == RESFMT_TABLE) 3142 cur_fmt = RESFMT_TABLE_CALCLEN; 3143 else 3144 cur_fmt = env.out_fmt; 3145 3146 one_more_time: 3147 output_comp_headers(cur_fmt); 3148 3149 last_idx = -1; 3150 cnt = 0; 3151 for (i = 0; i < env.join_stat_cnt; i++) { 3152 const struct verif_stats_join *join = &env.join_stats[i]; 3153 3154 if (!should_output_join_stats(join)) 3155 continue; 3156 3157 if (env.top_n && cnt >= env.top_n) 3158 break; 3159 3160 if (cur_fmt == RESFMT_TABLE_CALCLEN) 3161 last_idx = i; 3162 3163 output_comp_stats(join, cur_fmt, i == last_idx); 3164 3165 cnt++; 3166 } 3167 3168 if (cur_fmt == RESFMT_TABLE_CALCLEN) { 3169 cur_fmt = RESFMT_TABLE; 3170 goto one_more_time; /* ... this time with feeling */ 3171 } 3172 3173 return 0; 3174 } 3175 3176 static bool is_stat_filter_matched(struct filter *f, const struct verif_stats *stats) 3177 { 3178 long value = stats->stats[f->stat_id]; 3179 3180 if (f->abs) 3181 value = value < 0 ? -value : value; 3182 3183 switch (f->op) { 3184 case OP_EQ: return value == f->value; 3185 case OP_NEQ: return value != f->value; 3186 case OP_LT: return value < f->value; 3187 case OP_LE: return value <= f->value; 3188 case OP_GT: return value > f->value; 3189 case OP_GE: return value >= f->value; 3190 } 3191 3192 fprintf(stderr, "BUG: unknown filter op %d!\n", f->op); 3193 return false; 3194 } 3195 3196 static bool should_output_stats(const struct verif_stats *stats) 3197 { 3198 struct filter *f; 3199 int i, allow_cnt = 0; 3200 3201 for (i = 0; i < env.deny_filter_cnt; i++) { 3202 f = &env.deny_filters[i]; 3203 if (f->kind != FILTER_STAT) 3204 continue; 3205 3206 if (is_stat_filter_matched(f, stats)) 3207 return false; 3208 } 3209 3210 for (i = 0; i < env.allow_filter_cnt; i++) { 3211 f = &env.allow_filters[i]; 3212 if (f->kind != FILTER_STAT) 3213 continue; 3214 allow_cnt++; 3215 3216 if (is_stat_filter_matched(f, stats)) 3217 return true; 3218 } 3219 3220 /* if there are no stat allowed filters, pass everything through */ 3221 return allow_cnt == 0; 3222 } 3223 3224 static void output_prog_stats(void) 3225 { 3226 const struct verif_stats *stats; 3227 int i, last_stat_idx = 0, cnt = 0; 3228 3229 if (env.out_fmt == RESFMT_TABLE) { 3230 /* calculate column widths */ 3231 output_headers(RESFMT_TABLE_CALCLEN); 3232 for (i = 0; i < env.prog_stat_cnt; i++) { 3233 stats = &env.prog_stats[i]; 3234 if (!should_output_stats(stats)) 3235 continue; 3236 output_stats(stats, RESFMT_TABLE_CALCLEN, false); 3237 last_stat_idx = i; 3238 } 3239 } 3240 3241 /* actually output the table */ 3242 output_headers(env.out_fmt); 3243 for (i = 0; i < env.prog_stat_cnt; i++) { 3244 stats = &env.prog_stats[i]; 3245 if (!should_output_stats(stats)) 3246 continue; 3247 if (env.top_n && cnt >= env.top_n) 3248 break; 3249 output_stats(stats, env.out_fmt, i == last_stat_idx); 3250 cnt++; 3251 } 3252 } 3253 3254 static int handle_verif_mode(void) 3255 { 3256 int i, err = 0; 3257 3258 if (env.filename_cnt == 0) { 3259 fprintf(stderr, "Please provide path to BPF object file!\n\n"); 3260 argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); 3261 return -EINVAL; 3262 } 3263 3264 create_stat_cgroup(); 3265 for (i = 0; i < env.filename_cnt; i++) { 3266 err = process_obj(env.filenames[i]); 3267 if (err) { 3268 fprintf(stderr, "Failed to process '%s': %d\n", env.filenames[i], err); 3269 goto out; 3270 } 3271 } 3272 3273 qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats); 3274 3275 output_prog_stats(); 3276 3277 out: 3278 destroy_stat_cgroup(); 3279 return err; 3280 } 3281 3282 static int handle_replay_mode(void) 3283 { 3284 struct stat_specs specs = {}; 3285 int err; 3286 3287 if (env.filename_cnt != 1) { 3288 fprintf(stderr, "Replay mode expects exactly one input CSV file!\n\n"); 3289 argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); 3290 return -EINVAL; 3291 } 3292 3293 err = parse_stats_csv(env.filenames[0], &specs, 3294 &env.prog_stats, &env.prog_stat_cnt); 3295 if (err) { 3296 fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err); 3297 return err; 3298 } 3299 3300 qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats); 3301 3302 output_prog_stats(); 3303 3304 return 0; 3305 } 3306 3307 int main(int argc, char **argv) 3308 { 3309 int err = 0, i, j; 3310 3311 if (argp_parse(&argp, argc, argv, 0, NULL, NULL)) 3312 return 1; 3313 3314 if (env.show_version) { 3315 printf("%s\n", argp_program_version); 3316 return 0; 3317 } 3318 3319 if (env.verbose && env.quiet) { 3320 fprintf(stderr, "Verbose and quiet modes are incompatible, please specify just one or neither!\n\n"); 3321 argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); 3322 return 1; 3323 } 3324 if (env.verbose && env.log_level == 0) 3325 env.log_level = 1; 3326 3327 if (env.output_spec.spec_cnt == 0) { 3328 if (env.out_fmt == RESFMT_CSV) 3329 env.output_spec = default_csv_output_spec; 3330 else 3331 env.output_spec = default_output_spec; 3332 } 3333 if (env.sort_spec.spec_cnt == 0) 3334 env.sort_spec = default_sort_spec; 3335 3336 if (env.comparison_mode && env.replay_mode) { 3337 fprintf(stderr, "Can't specify replay and comparison mode at the same time!\n\n"); 3338 argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); 3339 return 1; 3340 } 3341 3342 if (env.comparison_mode) 3343 err = handle_comparison_mode(); 3344 else if (env.replay_mode) 3345 err = handle_replay_mode(); 3346 else 3347 err = handle_verif_mode(); 3348 3349 free_verif_stats(env.prog_stats, env.prog_stat_cnt); 3350 free_verif_stats(env.baseline_stats, env.baseline_stat_cnt); 3351 free(env.join_stats); 3352 for (i = 0; i < env.filename_cnt; i++) 3353 free(env.filenames[i]); 3354 free(env.filenames); 3355 for (i = 0; i < env.allow_filter_cnt; i++) { 3356 free(env.allow_filters[i].any_glob); 3357 free(env.allow_filters[i].file_glob); 3358 free(env.allow_filters[i].prog_glob); 3359 } 3360 free(env.allow_filters); 3361 for (i = 0; i < env.deny_filter_cnt; i++) { 3362 free(env.deny_filters[i].any_glob); 3363 free(env.deny_filters[i].file_glob); 3364 free(env.deny_filters[i].prog_glob); 3365 } 3366 free(env.deny_filters); 3367 for (i = 0; i < env.npresets; ++i) { 3368 free(env.presets[i].full_name); 3369 for (j = 0; j < env.presets[i].atom_count; ++j) { 3370 switch (env.presets[i].atoms[j].type) { 3371 case FIELD_NAME: 3372 free(env.presets[i].atoms[j].name); 3373 break; 3374 case ARRAY_INDEX: 3375 if (env.presets[i].atoms[j].index.type == ENUMERATOR) 3376 free(env.presets[i].atoms[j].index.svalue); 3377 break; 3378 } 3379 } 3380 free(env.presets[i].atoms); 3381 } 3382 free(env.presets); 3383 return -err; 3384 } 3385