xref: /linux/tools/testing/selftests/bpf/veristat.c (revision fb7399cf2d0b33825b8039f95c45395c7deba25c)
1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3 #define _GNU_SOURCE
4 #include <argp.h>
5 #include <libgen.h>
6 #include <ctype.h>
7 #include <string.h>
8 #include <stdlib.h>
9 #include <sched.h>
10 #include <pthread.h>
11 #include <dirent.h>
12 #include <signal.h>
13 #include <fcntl.h>
14 #include <unistd.h>
15 #include <sys/time.h>
16 #include <sys/sysinfo.h>
17 #include <sys/stat.h>
18 #include <bpf/libbpf.h>
19 #include <bpf/btf.h>
20 #include <bpf/bpf.h>
21 #include <libelf.h>
22 #include <gelf.h>
23 #include <float.h>
24 #include <math.h>
25 #include <limits.h>
26 
27 #ifndef ARRAY_SIZE
28 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
29 #endif
30 
31 #ifndef max
32 #define max(a, b) ((a) > (b) ? (a) : (b))
33 #endif
34 
35 #ifndef min
36 #define min(a, b) ((a) < (b) ? (a) : (b))
37 #endif
38 
39 enum stat_id {
40 	VERDICT,
41 	DURATION,
42 	TOTAL_INSNS,
43 	TOTAL_STATES,
44 	PEAK_STATES,
45 	MAX_STATES_PER_INSN,
46 	MARK_READ_MAX_LEN,
47 	SIZE,
48 	JITED_SIZE,
49 	STACK,
50 	PROG_TYPE,
51 	ATTACH_TYPE,
52 
53 	FILE_NAME,
54 	PROG_NAME,
55 
56 	ALL_STATS_CNT,
57 	NUM_STATS_CNT = FILE_NAME - VERDICT,
58 };
59 
60 /* In comparison mode each stat can specify up to four different values:
61  *   - A side value;
62  *   - B side value;
63  *   - absolute diff value;
64  *   - relative (percentage) diff value.
65  *
66  * When specifying stat specs in comparison mode, user can use one of the
67  * following variant suffixes to specify which exact variant should be used for
68  * ordering or filtering:
69  *   - `_a` for A side value;
70  *   - `_b` for B side value;
71  *   - `_diff` for absolute diff value;
72  *   - `_pct` for relative (percentage) diff value.
73  *
74  * If no variant suffix is provided, then `_b` (control data) is assumed.
75  *
76  * As an example, let's say instructions stat has the following output:
77  *
78  * Insns (A)  Insns (B)  Insns   (DIFF)
79  * ---------  ---------  --------------
80  * 21547      20920       -627 (-2.91%)
81  *
82  * Then:
83  *   - 21547 is A side value (insns_a);
84  *   - 20920 is B side value (insns_b);
85  *   - -627 is absolute diff value (insns_diff);
86  *   - -2.91% is relative diff value (insns_pct).
87  *
88  * For verdict there is no verdict_pct variant.
89  * For file and program name, _a and _b variants are equivalent and there are
90  * no _diff or _pct variants.
91  */
92 enum stat_variant {
93 	VARIANT_A,
94 	VARIANT_B,
95 	VARIANT_DIFF,
96 	VARIANT_PCT,
97 };
98 
99 struct verif_stats {
100 	char *file_name;
101 	char *prog_name;
102 
103 	long stats[NUM_STATS_CNT];
104 };
105 
106 /* joined comparison mode stats */
107 struct verif_stats_join {
108 	char *file_name;
109 	char *prog_name;
110 
111 	const struct verif_stats *stats_a;
112 	const struct verif_stats *stats_b;
113 };
114 
115 struct stat_specs {
116 	int spec_cnt;
117 	enum stat_id ids[ALL_STATS_CNT];
118 	enum stat_variant variants[ALL_STATS_CNT];
119 	bool asc[ALL_STATS_CNT];
120 	bool abs[ALL_STATS_CNT];
121 	int lens[ALL_STATS_CNT * 3]; /* 3x for comparison mode */
122 };
123 
124 enum resfmt {
125 	RESFMT_TABLE,
126 	RESFMT_TABLE_CALCLEN, /* fake format to pre-calculate table's column widths */
127 	RESFMT_CSV,
128 };
129 
130 enum filter_kind {
131 	FILTER_NAME,
132 	FILTER_STAT,
133 };
134 
135 enum operator_kind {
136 	OP_EQ,		/* == or = */
137 	OP_NEQ,		/* != or <> */
138 	OP_LT,		/* < */
139 	OP_LE,		/* <= */
140 	OP_GT,		/* > */
141 	OP_GE,		/* >= */
142 };
143 
144 struct filter {
145 	enum filter_kind kind;
146 	/* FILTER_NAME */
147 	char *any_glob;
148 	char *file_glob;
149 	char *prog_glob;
150 	/* FILTER_STAT */
151 	enum operator_kind op;
152 	int stat_id;
153 	enum stat_variant stat_var;
154 	long value;
155 	bool abs;
156 };
157 
158 struct var_preset {
159 	char *name;
160 	enum { INTEGRAL, ENUMERATOR } type;
161 	union {
162 		long long ivalue;
163 		char *svalue;
164 	};
165 	bool applied;
166 };
167 
168 static struct env {
169 	char **filenames;
170 	int filename_cnt;
171 	bool verbose;
172 	bool debug;
173 	bool quiet;
174 	bool force_checkpoints;
175 	bool force_reg_invariants;
176 	enum resfmt out_fmt;
177 	bool show_version;
178 	bool comparison_mode;
179 	bool replay_mode;
180 	int top_n;
181 
182 	int log_level;
183 	int log_size;
184 	bool log_fixed;
185 
186 	struct verif_stats *prog_stats;
187 	int prog_stat_cnt;
188 
189 	/* baseline_stats is allocated and used only in comparison mode */
190 	struct verif_stats *baseline_stats;
191 	int baseline_stat_cnt;
192 
193 	struct verif_stats_join *join_stats;
194 	int join_stat_cnt;
195 
196 	struct stat_specs output_spec;
197 	struct stat_specs sort_spec;
198 
199 	struct filter *allow_filters;
200 	struct filter *deny_filters;
201 	int allow_filter_cnt;
202 	int deny_filter_cnt;
203 
204 	int files_processed;
205 	int files_skipped;
206 	int progs_processed;
207 	int progs_skipped;
208 	int top_src_lines;
209 	struct var_preset *presets;
210 	int npresets;
211 } env;
212 
213 static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
214 {
215 	if (!env.verbose)
216 		return 0;
217 	if (level == LIBBPF_DEBUG  && !env.debug)
218 		return 0;
219 	return vfprintf(stderr, format, args);
220 }
221 
222 #ifndef VERISTAT_VERSION
223 #define VERISTAT_VERSION "<kernel>"
224 #endif
225 
226 const char *argp_program_version = "veristat v" VERISTAT_VERSION;
227 const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
228 const char argp_program_doc[] =
229 "veristat    BPF verifier stats collection and comparison tool.\n"
230 "\n"
231 "USAGE: veristat <obj-file> [<obj-file>...]\n"
232 "   OR: veristat -C <baseline.csv> <comparison.csv>\n"
233 "   OR: veristat -R <results.csv>\n"
234 "   OR: veristat -vl2 <to_analyze.bpf.o>\n";
235 
236 enum {
237 	OPT_LOG_FIXED = 1000,
238 	OPT_LOG_SIZE = 1001,
239 };
240 
241 static const struct argp_option opts[] = {
242 	{ NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help" },
243 	{ "version", 'V', NULL, 0, "Print version" },
244 	{ "verbose", 'v', NULL, 0, "Verbose mode" },
245 	{ "debug", 'd', NULL, 0, "Debug mode (turns on libbpf debug logging)" },
246 	{ "log-level", 'l', "LEVEL", 0, "Verifier log level (default 0 for normal mode, 1 for verbose mode, 2 for full verification log)" },
247 	{ "log-fixed", OPT_LOG_FIXED, NULL, 0, "Disable verifier log rotation" },
248 	{ "log-size", OPT_LOG_SIZE, "BYTES", 0, "Customize verifier log size (default to 16MB)" },
249 	{ "top-n", 'n', "N", 0, "Emit only up to first N results." },
250 	{ "quiet", 'q', NULL, 0, "Quiet mode" },
251 	{ "emit", 'e', "SPEC", 0, "Specify stats to be emitted" },
252 	{ "sort", 's', "SPEC", 0, "Specify sort order" },
253 	{ "output-format", 'o', "FMT", 0, "Result output format (table, csv), default is table." },
254 	{ "compare", 'C', NULL, 0, "Comparison mode" },
255 	{ "replay", 'R', NULL, 0, "Replay mode" },
256 	{ "filter", 'f', "FILTER", 0, "Filter expressions (or @filename for file with expressions)." },
257 	{ "test-states", 't', NULL, 0,
258 	  "Force frequent BPF verifier state checkpointing (set BPF_F_TEST_STATE_FREQ program flag)" },
259 	{ "test-reg-invariants", 'r', NULL, 0,
260 	  "Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" },
261 	{ "top-src-lines", 'S', "N", 0, "Emit N most frequent source code lines" },
262 	{ "set-global-vars", 'G', "GLOBAL", 0, "Set global variables provided in the expression, for example \"var1 = 1\"" },
263 	{},
264 };
265 
266 static int parse_stats(const char *stats_str, struct stat_specs *specs);
267 static int append_filter(struct filter **filters, int *cnt, const char *str);
268 static int append_filter_file(const char *path);
269 static int append_var_preset(struct var_preset **presets, int *cnt, const char *expr);
270 static int append_var_preset_file(const char *filename);
271 static int append_file(const char *path);
272 static int append_file_from_file(const char *path);
273 
274 static error_t parse_arg(int key, char *arg, struct argp_state *state)
275 {
276 	int err;
277 
278 	switch (key) {
279 	case 'h':
280 		argp_state_help(state, stderr, ARGP_HELP_STD_HELP);
281 		break;
282 	case 'V':
283 		env.show_version = true;
284 		break;
285 	case 'v':
286 		env.verbose = true;
287 		break;
288 	case 'd':
289 		env.debug = true;
290 		env.verbose = true;
291 		break;
292 	case 'q':
293 		env.quiet = true;
294 		break;
295 	case 'e':
296 		err = parse_stats(arg, &env.output_spec);
297 		if (err)
298 			return err;
299 		break;
300 	case 's':
301 		err = parse_stats(arg, &env.sort_spec);
302 		if (err)
303 			return err;
304 		break;
305 	case 'o':
306 		if (strcmp(arg, "table") == 0) {
307 			env.out_fmt = RESFMT_TABLE;
308 		} else if (strcmp(arg, "csv") == 0) {
309 			env.out_fmt = RESFMT_CSV;
310 		} else {
311 			fprintf(stderr, "Unrecognized output format '%s'\n", arg);
312 			return -EINVAL;
313 		}
314 		break;
315 	case 'l':
316 		errno = 0;
317 		env.log_level = strtol(arg, NULL, 10);
318 		if (errno) {
319 			fprintf(stderr, "invalid log level: %s\n", arg);
320 			argp_usage(state);
321 		}
322 		break;
323 	case OPT_LOG_FIXED:
324 		env.log_fixed = true;
325 		break;
326 	case OPT_LOG_SIZE:
327 		errno = 0;
328 		env.log_size = strtol(arg, NULL, 10);
329 		if (errno) {
330 			fprintf(stderr, "invalid log size: %s\n", arg);
331 			argp_usage(state);
332 		}
333 		break;
334 	case 't':
335 		env.force_checkpoints = true;
336 		break;
337 	case 'r':
338 		env.force_reg_invariants = true;
339 		break;
340 	case 'n':
341 		errno = 0;
342 		env.top_n = strtol(arg, NULL, 10);
343 		if (errno) {
344 			fprintf(stderr, "invalid top N specifier: %s\n", arg);
345 			argp_usage(state);
346 		}
347 	case 'C':
348 		env.comparison_mode = true;
349 		break;
350 	case 'R':
351 		env.replay_mode = true;
352 		break;
353 	case 'f':
354 		if (arg[0] == '@')
355 			err = append_filter_file(arg + 1);
356 		else if (arg[0] == '!')
357 			err = append_filter(&env.deny_filters, &env.deny_filter_cnt, arg + 1);
358 		else
359 			err = append_filter(&env.allow_filters, &env.allow_filter_cnt, arg);
360 		if (err) {
361 			fprintf(stderr, "Failed to collect program filter expressions: %d\n", err);
362 			return err;
363 		}
364 		break;
365 	case 'S':
366 		errno = 0;
367 		env.top_src_lines = strtol(arg, NULL, 10);
368 		if (errno) {
369 			fprintf(stderr, "invalid top lines N specifier: %s\n", arg);
370 			argp_usage(state);
371 		}
372 		break;
373 	case 'G': {
374 		if (arg[0] == '@')
375 			err = append_var_preset_file(arg + 1);
376 		else
377 			err = append_var_preset(&env.presets, &env.npresets, arg);
378 		if (err) {
379 			fprintf(stderr, "Failed to parse global variable presets: %s\n", arg);
380 			return err;
381 		}
382 		break;
383 	}
384 	case ARGP_KEY_ARG:
385 		if (arg[0] == '@')
386 			err = append_file_from_file(arg + 1);
387 		else
388 			err = append_file(arg);
389 		if (err) {
390 			fprintf(stderr, "Failed to collect BPF object files: %d\n", err);
391 			return err;
392 		}
393 		break;
394 	default:
395 		return ARGP_ERR_UNKNOWN;
396 	}
397 	return 0;
398 }
399 
400 static const struct argp argp = {
401 	.options = opts,
402 	.parser = parse_arg,
403 	.doc = argp_program_doc,
404 };
405 
406 
407 /* Adapted from perf/util/string.c */
408 static bool glob_matches(const char *str, const char *pat)
409 {
410 	while (*str && *pat && *pat != '*') {
411 		if (*str != *pat)
412 			return false;
413 		str++;
414 		pat++;
415 	}
416 	/* Check wild card */
417 	if (*pat == '*') {
418 		while (*pat == '*')
419 			pat++;
420 		if (!*pat) /* Tail wild card matches all */
421 			return true;
422 		while (*str)
423 			if (glob_matches(str++, pat))
424 				return true;
425 	}
426 	return !*str && !*pat;
427 }
428 
429 static bool is_bpf_obj_file(const char *path) {
430 	Elf64_Ehdr *ehdr;
431 	int fd, err = -EINVAL;
432 	Elf *elf = NULL;
433 
434 	fd = open(path, O_RDONLY | O_CLOEXEC);
435 	if (fd < 0)
436 		return true; /* we'll fail later and propagate error */
437 
438 	/* ensure libelf is initialized */
439 	(void)elf_version(EV_CURRENT);
440 
441 	elf = elf_begin(fd, ELF_C_READ, NULL);
442 	if (!elf)
443 		goto cleanup;
444 
445 	if (elf_kind(elf) != ELF_K_ELF || gelf_getclass(elf) != ELFCLASS64)
446 		goto cleanup;
447 
448 	ehdr = elf64_getehdr(elf);
449 	/* Old LLVM set e_machine to EM_NONE */
450 	if (!ehdr || ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF))
451 		goto cleanup;
452 
453 	err = 0;
454 cleanup:
455 	if (elf)
456 		elf_end(elf);
457 	close(fd);
458 	return err == 0;
459 }
460 
461 static bool should_process_file_prog(const char *filename, const char *prog_name)
462 {
463 	struct filter *f;
464 	int i, allow_cnt = 0;
465 
466 	for (i = 0; i < env.deny_filter_cnt; i++) {
467 		f = &env.deny_filters[i];
468 		if (f->kind != FILTER_NAME)
469 			continue;
470 
471 		if (f->any_glob && glob_matches(filename, f->any_glob))
472 			return false;
473 		if (f->any_glob && prog_name && glob_matches(prog_name, f->any_glob))
474 			return false;
475 		if (f->file_glob && glob_matches(filename, f->file_glob))
476 			return false;
477 		if (f->prog_glob && prog_name && glob_matches(prog_name, f->prog_glob))
478 			return false;
479 	}
480 
481 	for (i = 0; i < env.allow_filter_cnt; i++) {
482 		f = &env.allow_filters[i];
483 		if (f->kind != FILTER_NAME)
484 			continue;
485 
486 		allow_cnt++;
487 		if (f->any_glob) {
488 			if (glob_matches(filename, f->any_glob))
489 				return true;
490 			/* If we don't know program name yet, any_glob filter
491 			 * has to assume that current BPF object file might be
492 			 * relevant; we'll check again later on after opening
493 			 * BPF object file, at which point program name will
494 			 * be known finally.
495 			 */
496 			if (!prog_name || glob_matches(prog_name, f->any_glob))
497 				return true;
498 		} else {
499 			if (f->file_glob && !glob_matches(filename, f->file_glob))
500 				continue;
501 			if (f->prog_glob && prog_name && !glob_matches(prog_name, f->prog_glob))
502 				continue;
503 			return true;
504 		}
505 	}
506 
507 	/* if there are no file/prog name allow filters, allow all progs,
508 	 * unless they are denied earlier explicitly
509 	 */
510 	return allow_cnt == 0;
511 }
512 
513 static struct {
514 	enum operator_kind op_kind;
515 	const char *op_str;
516 } operators[] = {
517 	/* Order of these definitions matter to avoid situations like '<'
518 	 * matching part of what is actually a '<>' operator. That is,
519 	 * substrings should go last.
520 	 */
521 	{ OP_EQ, "==" },
522 	{ OP_NEQ, "!=" },
523 	{ OP_NEQ, "<>" },
524 	{ OP_LE, "<=" },
525 	{ OP_LT, "<" },
526 	{ OP_GE, ">=" },
527 	{ OP_GT, ">" },
528 	{ OP_EQ, "=" },
529 };
530 
531 static bool parse_stat_id_var(const char *name, size_t len, int *id,
532 			      enum stat_variant *var, bool *is_abs);
533 
534 static int append_filter(struct filter **filters, int *cnt, const char *str)
535 {
536 	struct filter *f;
537 	void *tmp;
538 	const char *p;
539 	int i;
540 
541 	tmp = realloc(*filters, (*cnt + 1) * sizeof(**filters));
542 	if (!tmp)
543 		return -ENOMEM;
544 	*filters = tmp;
545 
546 	f = &(*filters)[*cnt];
547 	memset(f, 0, sizeof(*f));
548 
549 	/* First, let's check if it's a stats filter of the following form:
550 	 * <stat><op><value, where:
551 	 *   - <stat> is one of supported numerical stats (verdict is also
552 	 *     considered numerical, failure == 0, success == 1);
553 	 *   - <op> is comparison operator (see `operators` definitions);
554 	 *   - <value> is an integer (or failure/success, or false/true as
555 	 *     special aliases for 0 and 1, respectively).
556 	 * If the form doesn't match what user provided, we assume file/prog
557 	 * glob filter.
558 	 */
559 	for (i = 0; i < ARRAY_SIZE(operators); i++) {
560 		enum stat_variant var;
561 		int id;
562 		long val;
563 		const char *end = str;
564 		const char *op_str;
565 		bool is_abs;
566 
567 		op_str = operators[i].op_str;
568 		p = strstr(str, op_str);
569 		if (!p)
570 			continue;
571 
572 		if (!parse_stat_id_var(str, p - str, &id, &var, &is_abs)) {
573 			fprintf(stderr, "Unrecognized stat name in '%s'!\n", str);
574 			return -EINVAL;
575 		}
576 		if (id >= FILE_NAME) {
577 			fprintf(stderr, "Non-integer stat is specified in '%s'!\n", str);
578 			return -EINVAL;
579 		}
580 
581 		p += strlen(op_str);
582 
583 		if (strcasecmp(p, "true") == 0 ||
584 		    strcasecmp(p, "t") == 0 ||
585 		    strcasecmp(p, "success") == 0 ||
586 		    strcasecmp(p, "succ") == 0 ||
587 		    strcasecmp(p, "s") == 0 ||
588 		    strcasecmp(p, "match") == 0 ||
589 		    strcasecmp(p, "m") == 0) {
590 			val = 1;
591 		} else if (strcasecmp(p, "false") == 0 ||
592 			   strcasecmp(p, "f") == 0 ||
593 			   strcasecmp(p, "failure") == 0 ||
594 			   strcasecmp(p, "fail") == 0 ||
595 			   strcasecmp(p, "mismatch") == 0 ||
596 			   strcasecmp(p, "mis") == 0) {
597 			val = 0;
598 		} else {
599 			errno = 0;
600 			val = strtol(p, (char **)&end, 10);
601 			if (errno || end == p || *end != '\0' ) {
602 				fprintf(stderr, "Invalid integer value in '%s'!\n", str);
603 				return -EINVAL;
604 			}
605 		}
606 
607 		f->kind = FILTER_STAT;
608 		f->stat_id = id;
609 		f->stat_var = var;
610 		f->op = operators[i].op_kind;
611 		f->abs = true;
612 		f->value = val;
613 
614 		*cnt += 1;
615 		return 0;
616 	}
617 
618 	/* File/prog filter can be specified either as '<glob>' or
619 	 * '<file-glob>/<prog-glob>'. In the former case <glob> is applied to
620 	 * both file and program names. This seems to be way more useful in
621 	 * practice. If user needs full control, they can use '/<prog-glob>'
622 	 * form to glob just program name, or '<file-glob>/' to glob only file
623 	 * name. But usually common <glob> seems to be the most useful and
624 	 * ergonomic way.
625 	 */
626 	f->kind = FILTER_NAME;
627 	p = strchr(str, '/');
628 	if (!p) {
629 		f->any_glob = strdup(str);
630 		if (!f->any_glob)
631 			return -ENOMEM;
632 	} else {
633 		if (str != p) {
634 			/* non-empty file glob */
635 			f->file_glob = strndup(str, p - str);
636 			if (!f->file_glob)
637 				return -ENOMEM;
638 		}
639 		if (strlen(p + 1) > 0) {
640 			/* non-empty prog glob */
641 			f->prog_glob = strdup(p + 1);
642 			if (!f->prog_glob) {
643 				free(f->file_glob);
644 				f->file_glob = NULL;
645 				return -ENOMEM;
646 			}
647 		}
648 	}
649 
650 	*cnt += 1;
651 	return 0;
652 }
653 
654 static int append_filter_file(const char *path)
655 {
656 	char buf[1024];
657 	FILE *f;
658 	int err = 0;
659 
660 	f = fopen(path, "r");
661 	if (!f) {
662 		err = -errno;
663 		fprintf(stderr, "Failed to open filters in '%s': %s\n", path, strerror(-err));
664 		return err;
665 	}
666 
667 	while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
668 		/* lines starting with # are comments, skip them */
669 		if (buf[0] == '\0' || buf[0] == '#')
670 			continue;
671 		/* lines starting with ! are negative match filters */
672 		if (buf[0] == '!')
673 			err = append_filter(&env.deny_filters, &env.deny_filter_cnt, buf + 1);
674 		else
675 			err = append_filter(&env.allow_filters, &env.allow_filter_cnt, buf);
676 		if (err)
677 			goto cleanup;
678 	}
679 
680 cleanup:
681 	fclose(f);
682 	return err;
683 }
684 
685 static const struct stat_specs default_output_spec = {
686 	.spec_cnt = 8,
687 	.ids = {
688 		FILE_NAME, PROG_NAME, VERDICT, DURATION,
689 		TOTAL_INSNS, TOTAL_STATES, SIZE, JITED_SIZE
690 	},
691 };
692 
693 static int append_file(const char *path)
694 {
695 	void *tmp;
696 
697 	tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));
698 	if (!tmp)
699 		return -ENOMEM;
700 	env.filenames = tmp;
701 	env.filenames[env.filename_cnt] = strdup(path);
702 	if (!env.filenames[env.filename_cnt])
703 		return -ENOMEM;
704 	env.filename_cnt++;
705 	return 0;
706 }
707 
708 static int append_file_from_file(const char *path)
709 {
710 	char buf[1024];
711 	int err = 0;
712 	FILE *f;
713 
714 	f = fopen(path, "r");
715 	if (!f) {
716 		err = -errno;
717 		fprintf(stderr, "Failed to open object files list in '%s': %s\n",
718 			path, strerror(errno));
719 		return err;
720 	}
721 
722 	while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
723 		/* lines starting with # are comments, skip them */
724 		if (buf[0] == '\0' || buf[0] == '#')
725 			continue;
726 		err = append_file(buf);
727 		if (err)
728 			goto cleanup;
729 	}
730 
731 cleanup:
732 	fclose(f);
733 	return err;
734 }
735 
736 static const struct stat_specs default_csv_output_spec = {
737 	.spec_cnt = 14,
738 	.ids = {
739 		FILE_NAME, PROG_NAME, VERDICT, DURATION,
740 		TOTAL_INSNS, TOTAL_STATES, PEAK_STATES,
741 		MAX_STATES_PER_INSN, MARK_READ_MAX_LEN,
742 		SIZE, JITED_SIZE, PROG_TYPE, ATTACH_TYPE,
743 		STACK,
744 	},
745 };
746 
747 static const struct stat_specs default_sort_spec = {
748 	.spec_cnt = 2,
749 	.ids = {
750 		FILE_NAME, PROG_NAME,
751 	},
752 	.asc = { true, true, },
753 };
754 
755 /* sorting for comparison mode to join two data sets */
756 static const struct stat_specs join_sort_spec = {
757 	.spec_cnt = 2,
758 	.ids = {
759 		FILE_NAME, PROG_NAME,
760 	},
761 	.asc = { true, true, },
762 };
763 
764 static struct stat_def {
765 	const char *header;
766 	const char *names[4];
767 	bool asc_by_default;
768 	bool left_aligned;
769 } stat_defs[] = {
770 	[FILE_NAME] = { "File", {"file_name", "filename", "file"}, true /* asc */, true /* left */ },
771 	[PROG_NAME] = { "Program", {"prog_name", "progname", "prog"}, true /* asc */, true /* left */ },
772 	[VERDICT] = { "Verdict", {"verdict"}, true /* asc: failure, success */, true /* left */ },
773 	[DURATION] = { "Duration (us)", {"duration", "dur"}, },
774 	[TOTAL_INSNS] = { "Insns", {"total_insns", "insns"}, },
775 	[TOTAL_STATES] = { "States", {"total_states", "states"}, },
776 	[PEAK_STATES] = { "Peak states", {"peak_states"}, },
777 	[MAX_STATES_PER_INSN] = { "Max states per insn", {"max_states_per_insn"}, },
778 	[MARK_READ_MAX_LEN] = { "Max mark read length", {"max_mark_read_len", "mark_read"}, },
779 	[SIZE] = { "Program size", {"prog_size"}, },
780 	[JITED_SIZE] = { "Jited size", {"prog_size_jited"}, },
781 	[STACK] = {"Stack depth", {"stack_depth", "stack"}, },
782 	[PROG_TYPE] = { "Program type", {"prog_type"}, },
783 	[ATTACH_TYPE] = { "Attach type", {"attach_type", }, },
784 };
785 
786 static bool parse_stat_id_var(const char *name, size_t len, int *id,
787 			      enum stat_variant *var, bool *is_abs)
788 {
789 	static const char *var_sfxs[] = {
790 		[VARIANT_A] = "_a",
791 		[VARIANT_B] = "_b",
792 		[VARIANT_DIFF] = "_diff",
793 		[VARIANT_PCT] = "_pct",
794 	};
795 	int i, j, k;
796 
797 	/* |<stat>| means we take absolute value of given stat */
798 	*is_abs = false;
799 	if (len > 2 && name[0] == '|' && name[len - 1] == '|') {
800 		*is_abs = true;
801 		name += 1;
802 		len -= 2;
803 	}
804 
805 	for (i = 0; i < ARRAY_SIZE(stat_defs); i++) {
806 		struct stat_def *def = &stat_defs[i];
807 		size_t alias_len, sfx_len;
808 		const char *alias;
809 
810 		for (j = 0; j < ARRAY_SIZE(stat_defs[i].names); j++) {
811 			alias = def->names[j];
812 			if (!alias)
813 				continue;
814 
815 			alias_len = strlen(alias);
816 			if (strncmp(name, alias, alias_len) != 0)
817 				continue;
818 
819 			if (alias_len == len) {
820 				/* If no variant suffix is specified, we
821 				 * assume control group (just in case we are
822 				 * in comparison mode. Variant is ignored in
823 				 * non-comparison mode.
824 				 */
825 				*var = VARIANT_B;
826 				*id = i;
827 				return true;
828 			}
829 
830 			for (k = 0; k < ARRAY_SIZE(var_sfxs); k++) {
831 				sfx_len = strlen(var_sfxs[k]);
832 				if (alias_len + sfx_len != len)
833 					continue;
834 
835 				if (strncmp(name + alias_len, var_sfxs[k], sfx_len) == 0) {
836 					*var = (enum stat_variant)k;
837 					*id = i;
838 					return true;
839 				}
840 			}
841 		}
842 	}
843 
844 	return false;
845 }
846 
847 static bool is_asc_sym(char c)
848 {
849 	return c == '^';
850 }
851 
852 static bool is_desc_sym(char c)
853 {
854 	return c == 'v' || c == 'V' || c == '.' || c == '!' || c == '_';
855 }
856 
857 static int parse_stat(const char *stat_name, struct stat_specs *specs)
858 {
859 	int id;
860 	bool has_order = false, is_asc = false, is_abs = false;
861 	size_t len = strlen(stat_name);
862 	enum stat_variant var;
863 
864 	if (specs->spec_cnt >= ARRAY_SIZE(specs->ids)) {
865 		fprintf(stderr, "Can't specify more than %zd stats\n", ARRAY_SIZE(specs->ids));
866 		return -E2BIG;
867 	}
868 
869 	if (len > 1 && (is_asc_sym(stat_name[len - 1]) || is_desc_sym(stat_name[len - 1]))) {
870 		has_order = true;
871 		is_asc = is_asc_sym(stat_name[len - 1]);
872 		len -= 1;
873 	}
874 
875 	if (!parse_stat_id_var(stat_name, len, &id, &var, &is_abs)) {
876 		fprintf(stderr, "Unrecognized stat name '%s'\n", stat_name);
877 		return -ESRCH;
878 	}
879 
880 	specs->ids[specs->spec_cnt] = id;
881 	specs->variants[specs->spec_cnt] = var;
882 	specs->asc[specs->spec_cnt] = has_order ? is_asc : stat_defs[id].asc_by_default;
883 	specs->abs[specs->spec_cnt] = is_abs;
884 	specs->spec_cnt++;
885 
886 	return 0;
887 }
888 
889 static int parse_stats(const char *stats_str, struct stat_specs *specs)
890 {
891 	char *input, *state = NULL, *next;
892 	int err, cnt = 0;
893 
894 	input = strdup(stats_str);
895 	if (!input)
896 		return -ENOMEM;
897 
898 	while ((next = strtok_r(cnt++ ? NULL : input, ",", &state))) {
899 		err = parse_stat(next, specs);
900 		if (err) {
901 			free(input);
902 			return err;
903 		}
904 	}
905 
906 	free(input);
907 	return 0;
908 }
909 
910 static void free_verif_stats(struct verif_stats *stats, size_t stat_cnt)
911 {
912 	int i;
913 
914 	if (!stats)
915 		return;
916 
917 	for (i = 0; i < stat_cnt; i++) {
918 		free(stats[i].file_name);
919 		free(stats[i].prog_name);
920 	}
921 	free(stats);
922 }
923 
924 static char verif_log_buf[64 * 1024];
925 
926 #define MAX_PARSED_LOG_LINES 100
927 
928 static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats *s)
929 {
930 	const char *cur;
931 	int pos, lines, sub_stack, cnt = 0;
932 	char *state = NULL, *token, stack[512];
933 
934 	buf[buf_sz - 1] = '\0';
935 
936 	for (pos = strlen(buf) - 1, lines = 0; pos >= 0 && lines < MAX_PARSED_LOG_LINES; lines++) {
937 		/* find previous endline or otherwise take the start of log buf */
938 		for (cur = &buf[pos]; cur > buf && cur[0] != '\n'; cur--, pos--) {
939 		}
940 		/* next time start from end of previous line (or pos goes to <0) */
941 		pos--;
942 		/* if we found endline, point right after endline symbol;
943 		 * otherwise, stay at the beginning of log buf
944 		 */
945 		if (cur[0] == '\n')
946 			cur++;
947 
948 		if (1 == sscanf(cur, "verification time %ld usec\n", &s->stats[DURATION]))
949 			continue;
950 		if (5 == sscanf(cur, "processed %ld insns (limit %*d) max_states_per_insn %ld total_states %ld peak_states %ld mark_read %ld",
951 				&s->stats[TOTAL_INSNS],
952 				&s->stats[MAX_STATES_PER_INSN],
953 				&s->stats[TOTAL_STATES],
954 				&s->stats[PEAK_STATES],
955 				&s->stats[MARK_READ_MAX_LEN]))
956 			continue;
957 
958 		if (1 == sscanf(cur, "stack depth %511s", stack))
959 			continue;
960 	}
961 	while ((token = strtok_r(cnt++ ? NULL : stack, "+", &state))) {
962 		if (sscanf(token, "%d", &sub_stack) == 0)
963 			break;
964 		s->stats[STACK] += sub_stack;
965 	}
966 	return 0;
967 }
968 
969 struct line_cnt {
970 	char *line;
971 	int cnt;
972 };
973 
974 static int str_cmp(const void *a, const void *b)
975 {
976 	const char **str1 = (const char **)a;
977 	const char **str2 = (const char **)b;
978 
979 	return strcmp(*str1, *str2);
980 }
981 
982 static int line_cnt_cmp(const void *a, const void *b)
983 {
984 	const struct line_cnt *a_cnt = (const struct line_cnt *)a;
985 	const struct line_cnt *b_cnt = (const struct line_cnt *)b;
986 
987 	if (a_cnt->cnt != b_cnt->cnt)
988 		return a_cnt->cnt > b_cnt->cnt ? -1 : 1;
989 	return strcmp(a_cnt->line, b_cnt->line);
990 }
991 
992 static int print_top_src_lines(char * const buf, size_t buf_sz, const char *prog_name)
993 {
994 	int lines_cap = 0;
995 	int lines_size = 0;
996 	char **lines = NULL;
997 	char *line = NULL;
998 	char *state;
999 	struct line_cnt *freq = NULL;
1000 	struct line_cnt *cur;
1001 	int unique_lines;
1002 	int err = 0;
1003 	int i;
1004 
1005 	while ((line = strtok_r(line ? NULL : buf, "\n", &state))) {
1006 		if (strncmp(line, "; ", 2) != 0)
1007 			continue;
1008 		line += 2;
1009 
1010 		if (lines_size == lines_cap) {
1011 			char **tmp;
1012 
1013 			lines_cap = max(16, lines_cap * 2);
1014 			tmp = realloc(lines, lines_cap * sizeof(*tmp));
1015 			if (!tmp) {
1016 				err = -ENOMEM;
1017 				goto cleanup;
1018 			}
1019 			lines = tmp;
1020 		}
1021 		lines[lines_size] = line;
1022 		lines_size++;
1023 	}
1024 
1025 	if (lines_size == 0)
1026 		goto cleanup;
1027 
1028 	qsort(lines, lines_size, sizeof(*lines), str_cmp);
1029 
1030 	freq = calloc(lines_size, sizeof(*freq));
1031 	if (!freq) {
1032 		err = -ENOMEM;
1033 		goto cleanup;
1034 	}
1035 
1036 	cur = freq;
1037 	cur->line = lines[0];
1038 	cur->cnt = 1;
1039 	for (i = 1; i < lines_size; ++i) {
1040 		if (strcmp(lines[i], cur->line) != 0) {
1041 			cur++;
1042 			cur->line = lines[i];
1043 			cur->cnt = 0;
1044 		}
1045 		cur->cnt++;
1046 	}
1047 	unique_lines = cur - freq + 1;
1048 
1049 	qsort(freq, unique_lines, sizeof(struct line_cnt), line_cnt_cmp);
1050 
1051 	printf("Top source lines (%s):\n", prog_name);
1052 	for (i = 0; i < min(unique_lines, env.top_src_lines); ++i) {
1053 		const char *src_code = freq[i].line;
1054 		const char *src_line = NULL;
1055 		char *split = strrchr(freq[i].line, '@');
1056 
1057 		if (split) {
1058 			src_line = split + 1;
1059 
1060 			while (*src_line && isspace(*src_line))
1061 				src_line++;
1062 
1063 			while (split > src_code && isspace(*split))
1064 				split--;
1065 			*split = '\0';
1066 		}
1067 
1068 		if (src_line)
1069 			printf("%5d: (%s)\t%s\n", freq[i].cnt, src_line, src_code);
1070 		else
1071 			printf("%5d: %s\n", freq[i].cnt, src_code);
1072 	}
1073 	printf("\n");
1074 
1075 cleanup:
1076 	free(freq);
1077 	free(lines);
1078 	return err;
1079 }
1080 
1081 static int guess_prog_type_by_ctx_name(const char *ctx_name,
1082 				       enum bpf_prog_type *prog_type,
1083 				       enum bpf_attach_type *attach_type)
1084 {
1085 	/* We need to guess program type based on its declared context type.
1086 	 * This guess can't be perfect as many different program types might
1087 	 * share the same context type.  So we can only hope to reasonably
1088 	 * well guess this and get lucky.
1089 	 *
1090 	 * Just in case, we support both UAPI-side type names and
1091 	 * kernel-internal names.
1092 	 */
1093 	static struct {
1094 		const char *uapi_name;
1095 		const char *kern_name;
1096 		enum bpf_prog_type prog_type;
1097 		enum bpf_attach_type attach_type;
1098 	} ctx_map[] = {
1099 		/* __sk_buff is most ambiguous, we assume TC program */
1100 		{ "__sk_buff", "sk_buff", BPF_PROG_TYPE_SCHED_CLS },
1101 		{ "bpf_sock", "sock", BPF_PROG_TYPE_CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND },
1102 		{ "bpf_sock_addr", "bpf_sock_addr_kern",  BPF_PROG_TYPE_CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND },
1103 		{ "bpf_sock_ops", "bpf_sock_ops_kern", BPF_PROG_TYPE_SOCK_OPS, BPF_CGROUP_SOCK_OPS },
1104 		{ "sk_msg_md", "sk_msg", BPF_PROG_TYPE_SK_MSG, BPF_SK_MSG_VERDICT },
1105 		{ "bpf_cgroup_dev_ctx", "bpf_cgroup_dev_ctx", BPF_PROG_TYPE_CGROUP_DEVICE, BPF_CGROUP_DEVICE },
1106 		{ "bpf_sysctl", "bpf_sysctl_kern", BPF_PROG_TYPE_CGROUP_SYSCTL, BPF_CGROUP_SYSCTL },
1107 		{ "bpf_sockopt", "bpf_sockopt_kern", BPF_PROG_TYPE_CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT },
1108 		{ "sk_reuseport_md", "sk_reuseport_kern", BPF_PROG_TYPE_SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE },
1109 		{ "bpf_sk_lookup", "bpf_sk_lookup_kern", BPF_PROG_TYPE_SK_LOOKUP, BPF_SK_LOOKUP },
1110 		{ "xdp_md", "xdp_buff", BPF_PROG_TYPE_XDP, BPF_XDP },
1111 		/* tracing types with no expected attach type */
1112 		{ "bpf_user_pt_regs_t", "pt_regs", BPF_PROG_TYPE_KPROBE },
1113 		{ "bpf_perf_event_data", "bpf_perf_event_data_kern", BPF_PROG_TYPE_PERF_EVENT },
1114 		/* raw_tp programs use u64[] from kernel side, we don't want
1115 		 * to match on that, probably; so NULL for kern-side type
1116 		 */
1117 		{ "bpf_raw_tracepoint_args", NULL, BPF_PROG_TYPE_RAW_TRACEPOINT },
1118 	};
1119 	int i;
1120 
1121 	if (!ctx_name)
1122 		return -EINVAL;
1123 
1124 	for (i = 0; i < ARRAY_SIZE(ctx_map); i++) {
1125 		if (strcmp(ctx_map[i].uapi_name, ctx_name) == 0 ||
1126 		    (ctx_map[i].kern_name && strcmp(ctx_map[i].kern_name, ctx_name) == 0)) {
1127 			*prog_type = ctx_map[i].prog_type;
1128 			*attach_type = ctx_map[i].attach_type;
1129 			return 0;
1130 		}
1131 	}
1132 
1133 	return -ESRCH;
1134 }
1135 
1136 /* Make sure only target program is referenced from struct_ops map,
1137  * otherwise libbpf would automatically set autocreate for all
1138  * referenced programs.
1139  * See libbpf.c:bpf_object_adjust_struct_ops_autoload.
1140  */
1141 static void mask_unrelated_struct_ops_progs(struct bpf_object *obj,
1142 					    struct bpf_map *map,
1143 					    struct bpf_program *prog)
1144 {
1145 	struct btf *btf = bpf_object__btf(obj);
1146 	const struct btf_type *t, *mt;
1147 	struct btf_member *m;
1148 	int i, moff;
1149 	size_t data_sz, ptr_sz = sizeof(void *);
1150 	void *data;
1151 
1152 	t = btf__type_by_id(btf, bpf_map__btf_value_type_id(map));
1153 	if (!btf_is_struct(t))
1154 		return;
1155 
1156 	data = bpf_map__initial_value(map, &data_sz);
1157 	for (i = 0; i < btf_vlen(t); i++) {
1158 		m = &btf_members(t)[i];
1159 		mt = btf__type_by_id(btf, m->type);
1160 		if (!btf_is_ptr(mt))
1161 			continue;
1162 		moff = m->offset / 8;
1163 		if (moff + ptr_sz > data_sz)
1164 			continue;
1165 		if (memcmp(data + moff, &prog, ptr_sz) == 0)
1166 			continue;
1167 		memset(data + moff, 0, ptr_sz);
1168 	}
1169 }
1170 
1171 static void fixup_obj(struct bpf_object *obj, struct bpf_program *prog, const char *filename)
1172 {
1173 	struct bpf_map *map;
1174 
1175 	bpf_object__for_each_map(map, obj) {
1176 		/* disable pinning */
1177 		bpf_map__set_pin_path(map, NULL);
1178 
1179 		/* fix up map size, if necessary */
1180 		switch (bpf_map__type(map)) {
1181 		case BPF_MAP_TYPE_SK_STORAGE:
1182 		case BPF_MAP_TYPE_TASK_STORAGE:
1183 		case BPF_MAP_TYPE_INODE_STORAGE:
1184 		case BPF_MAP_TYPE_CGROUP_STORAGE:
1185 			break;
1186 		case BPF_MAP_TYPE_STRUCT_OPS:
1187 			mask_unrelated_struct_ops_progs(obj, map, prog);
1188 			break;
1189 		default:
1190 			if (bpf_map__max_entries(map) == 0)
1191 				bpf_map__set_max_entries(map, 1);
1192 		}
1193 	}
1194 
1195 	/* SEC(freplace) programs can't be loaded with veristat as is,
1196 	 * but we can try guessing their target program's expected type by
1197 	 * looking at the type of program's first argument and substituting
1198 	 * corresponding program type
1199 	 */
1200 	if (bpf_program__type(prog) == BPF_PROG_TYPE_EXT) {
1201 		const struct btf *btf = bpf_object__btf(obj);
1202 		const char *prog_name = bpf_program__name(prog);
1203 		enum bpf_prog_type prog_type;
1204 		enum bpf_attach_type attach_type;
1205 		const struct btf_type *t;
1206 		const char *ctx_name;
1207 		int id;
1208 
1209 		if (!btf)
1210 			goto skip_freplace_fixup;
1211 
1212 		id = btf__find_by_name_kind(btf, prog_name, BTF_KIND_FUNC);
1213 		t = btf__type_by_id(btf, id);
1214 		t = btf__type_by_id(btf, t->type);
1215 		if (!btf_is_func_proto(t) || btf_vlen(t) != 1)
1216 			goto skip_freplace_fixup;
1217 
1218 		/* context argument is a pointer to a struct/typedef */
1219 		t = btf__type_by_id(btf, btf_params(t)[0].type);
1220 		while (t && btf_is_mod(t))
1221 			t = btf__type_by_id(btf, t->type);
1222 		if (!t || !btf_is_ptr(t))
1223 			goto skip_freplace_fixup;
1224 		t = btf__type_by_id(btf, t->type);
1225 		while (t && btf_is_mod(t))
1226 			t = btf__type_by_id(btf, t->type);
1227 		if (!t)
1228 			goto skip_freplace_fixup;
1229 
1230 		ctx_name = btf__name_by_offset(btf, t->name_off);
1231 
1232 		if (guess_prog_type_by_ctx_name(ctx_name, &prog_type, &attach_type) == 0) {
1233 			bpf_program__set_type(prog, prog_type);
1234 			bpf_program__set_expected_attach_type(prog, attach_type);
1235 
1236 			if (!env.quiet) {
1237 				fprintf(stderr, "Using guessed program type '%s' for %s/%s...\n",
1238 					libbpf_bpf_prog_type_str(prog_type),
1239 					filename, prog_name);
1240 			}
1241 		} else {
1242 			if (!env.quiet) {
1243 				fprintf(stderr, "Failed to guess program type for freplace program with context type name '%s' for %s/%s. Consider using canonical type names to help veristat...\n",
1244 					ctx_name, filename, prog_name);
1245 			}
1246 		}
1247 	}
1248 skip_freplace_fixup:
1249 	return;
1250 }
1251 
1252 static int max_verifier_log_size(void)
1253 {
1254 	const int SMALL_LOG_SIZE = UINT_MAX >> 8;
1255 	const int BIG_LOG_SIZE = UINT_MAX >> 2;
1256 	struct bpf_insn insns[] = {
1257 		{ .code = BPF_ALU | BPF_MOV | BPF_X, .dst_reg = BPF_REG_0, },
1258 		{ .code  = BPF_JMP | BPF_EXIT, },
1259 	};
1260 	LIBBPF_OPTS(bpf_prog_load_opts, opts,
1261 		    .log_size = BIG_LOG_SIZE,
1262 		    .log_buf = (void *)-1,
1263 		    .log_level = 4
1264 	);
1265 	int ret, insn_cnt = ARRAY_SIZE(insns);
1266 	static int log_size;
1267 
1268 	if (log_size != 0)
1269 		return log_size;
1270 
1271 	ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts);
1272 
1273 	if (ret == -EFAULT)
1274 		log_size = BIG_LOG_SIZE;
1275 	else /* ret == -EINVAL, big log size is not supported by the verifier */
1276 		log_size = SMALL_LOG_SIZE;
1277 
1278 	return log_size;
1279 }
1280 
1281 static int process_prog(const char *filename, struct bpf_object *obj, struct bpf_program *prog)
1282 {
1283 	const char *base_filename = basename(strdupa(filename));
1284 	const char *prog_name = bpf_program__name(prog);
1285 	char *buf;
1286 	int buf_sz, log_level;
1287 	struct verif_stats *stats;
1288 	struct bpf_prog_info info;
1289 	__u32 info_len = sizeof(info);
1290 	int err = 0;
1291 	void *tmp;
1292 	int fd;
1293 
1294 	if (!should_process_file_prog(base_filename, bpf_program__name(prog))) {
1295 		env.progs_skipped++;
1296 		return 0;
1297 	}
1298 
1299 	tmp = realloc(env.prog_stats, (env.prog_stat_cnt + 1) * sizeof(*env.prog_stats));
1300 	if (!tmp)
1301 		return -ENOMEM;
1302 	env.prog_stats = tmp;
1303 	stats = &env.prog_stats[env.prog_stat_cnt++];
1304 	memset(stats, 0, sizeof(*stats));
1305 
1306 	if (env.verbose || env.top_src_lines > 0) {
1307 		buf_sz = env.log_size ? env.log_size : max_verifier_log_size();
1308 		buf = malloc(buf_sz);
1309 		if (!buf)
1310 			return -ENOMEM;
1311 		/* ensure we always request stats */
1312 		log_level = env.log_level | 4 | (env.log_fixed ? 8 : 0);
1313 		/* --top-src-lines needs verifier log */
1314 		if (env.top_src_lines > 0 && env.log_level == 0)
1315 			log_level |= 2;
1316 	} else {
1317 		buf = verif_log_buf;
1318 		buf_sz = sizeof(verif_log_buf);
1319 		/* request only verifier stats */
1320 		log_level = 4 | (env.log_fixed ? 8 : 0);
1321 	}
1322 	verif_log_buf[0] = '\0';
1323 
1324 	bpf_program__set_log_buf(prog, buf, buf_sz);
1325 	bpf_program__set_log_level(prog, log_level);
1326 
1327 	/* increase chances of successful BPF object loading */
1328 	fixup_obj(obj, prog, base_filename);
1329 
1330 	if (env.force_checkpoints)
1331 		bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_STATE_FREQ);
1332 	if (env.force_reg_invariants)
1333 		bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_REG_INVARIANTS);
1334 
1335 	err = bpf_object__load(obj);
1336 	env.progs_processed++;
1337 
1338 	stats->file_name = strdup(base_filename);
1339 	stats->prog_name = strdup(bpf_program__name(prog));
1340 	stats->stats[VERDICT] = err == 0; /* 1 - success, 0 - failure */
1341 	stats->stats[SIZE] = bpf_program__insn_cnt(prog);
1342 	stats->stats[PROG_TYPE] = bpf_program__type(prog);
1343 	stats->stats[ATTACH_TYPE] = bpf_program__expected_attach_type(prog);
1344 
1345 	memset(&info, 0, info_len);
1346 	fd = bpf_program__fd(prog);
1347 	if (fd > 0 && bpf_prog_get_info_by_fd(fd, &info, &info_len) == 0)
1348 		stats->stats[JITED_SIZE] = info.jited_prog_len;
1349 
1350 	parse_verif_log(buf, buf_sz, stats);
1351 
1352 	if (env.verbose) {
1353 		printf("PROCESSING %s/%s, DURATION US: %ld, VERDICT: %s, VERIFIER LOG:\n%s\n",
1354 		       filename, prog_name, stats->stats[DURATION],
1355 		       err ? "failure" : "success", buf);
1356 	}
1357 	if (env.top_src_lines > 0)
1358 		print_top_src_lines(buf, buf_sz, stats->prog_name);
1359 
1360 	if (verif_log_buf != buf)
1361 		free(buf);
1362 
1363 	return 0;
1364 };
1365 
1366 static int append_var_preset(struct var_preset **presets, int *cnt, const char *expr)
1367 {
1368 	void *tmp;
1369 	struct var_preset *cur;
1370 	char var[256], val[256], *val_end;
1371 	long long value;
1372 	int n;
1373 
1374 	tmp = realloc(*presets, (*cnt + 1) * sizeof(**presets));
1375 	if (!tmp)
1376 		return -ENOMEM;
1377 	*presets = tmp;
1378 	cur = &(*presets)[*cnt];
1379 	memset(cur, 0, sizeof(*cur));
1380 	(*cnt)++;
1381 
1382 	if (sscanf(expr, "%s = %s %n", var, val, &n) != 2 || n != strlen(expr)) {
1383 		fprintf(stderr, "Failed to parse expression '%s'\n", expr);
1384 		return -EINVAL;
1385 	}
1386 
1387 	if (val[0] == '-' || isdigit(val[0])) {
1388 		/* must be a number */
1389 		errno = 0;
1390 		value = strtoll(val, &val_end, 0);
1391 		if (errno == ERANGE) {
1392 			errno = 0;
1393 			value = strtoull(val, &val_end, 0);
1394 		}
1395 		if (errno || *val_end != '\0') {
1396 			fprintf(stderr, "Failed to parse value '%s'\n", val);
1397 			return -EINVAL;
1398 		}
1399 		cur->ivalue = value;
1400 		cur->type = INTEGRAL;
1401 	} else {
1402 		/* if not a number, consider it enum value */
1403 		cur->svalue = strdup(val);
1404 		if (!cur->svalue)
1405 			return -ENOMEM;
1406 		cur->type = ENUMERATOR;
1407 	}
1408 
1409 	cur->name = strdup(var);
1410 	if (!cur->name)
1411 		return -ENOMEM;
1412 
1413 	return 0;
1414 }
1415 
1416 static int append_var_preset_file(const char *filename)
1417 {
1418 	char buf[1024];
1419 	FILE *f;
1420 	int err = 0;
1421 
1422 	f = fopen(filename, "rt");
1423 	if (!f) {
1424 		err = -errno;
1425 		fprintf(stderr, "Failed to open presets in '%s': %s\n", filename, strerror(-err));
1426 		return -EINVAL;
1427 	}
1428 
1429 	while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
1430 		if (buf[0] == '\0' || buf[0] == '#')
1431 			continue;
1432 
1433 		err = append_var_preset(&env.presets, &env.npresets, buf);
1434 		if (err)
1435 			goto cleanup;
1436 	}
1437 
1438 cleanup:
1439 	fclose(f);
1440 	return err;
1441 }
1442 
1443 static bool is_signed_type(const struct btf_type *t)
1444 {
1445 	if (btf_is_int(t))
1446 		return btf_int_encoding(t) & BTF_INT_SIGNED;
1447 	if (btf_is_any_enum(t))
1448 		return btf_kflag(t);
1449 	return true;
1450 }
1451 
1452 static int enum_value_from_name(const struct btf *btf, const struct btf_type *t,
1453 				const char *evalue, long long *retval)
1454 {
1455 	if (btf_is_enum(t)) {
1456 		struct btf_enum *e = btf_enum(t);
1457 		int i, n = btf_vlen(t);
1458 
1459 		for (i = 0; i < n; ++i, ++e) {
1460 			const char *cur_name = btf__name_by_offset(btf, e->name_off);
1461 
1462 			if (strcmp(cur_name, evalue) == 0) {
1463 				*retval = e->val;
1464 				return 0;
1465 			}
1466 		}
1467 	} else if (btf_is_enum64(t)) {
1468 		struct btf_enum64 *e = btf_enum64(t);
1469 		int i, n = btf_vlen(t);
1470 
1471 		for (i = 0; i < n; ++i, ++e) {
1472 			const char *cur_name = btf__name_by_offset(btf, e->name_off);
1473 			__u64 value =  btf_enum64_value(e);
1474 
1475 			if (strcmp(cur_name, evalue) == 0) {
1476 				*retval = value;
1477 				return 0;
1478 			}
1479 		}
1480 	}
1481 	return -EINVAL;
1482 }
1483 
1484 static bool is_preset_supported(const struct btf_type *t)
1485 {
1486 	return btf_is_int(t) || btf_is_enum(t) || btf_is_enum64(t);
1487 }
1488 
1489 const int btf_find_member(const struct btf *btf,
1490 			  const struct btf_type *parent_type,
1491 			  __u32 parent_offset,
1492 			  const char *member_name,
1493 			  int *member_tid,
1494 			  __u32 *member_offset)
1495 {
1496 	int i;
1497 
1498 	if (!btf_is_composite(parent_type))
1499 		return -EINVAL;
1500 
1501 	for (i = 0; i < btf_vlen(parent_type); ++i) {
1502 		const struct btf_member *member;
1503 		const struct btf_type *member_type;
1504 		int tid;
1505 
1506 		member = btf_members(parent_type) + i;
1507 		tid =  btf__resolve_type(btf, member->type);
1508 		if (tid < 0)
1509 			return -EINVAL;
1510 
1511 		member_type = btf__type_by_id(btf, tid);
1512 		if (member->name_off) {
1513 			const char *name = btf__name_by_offset(btf, member->name_off);
1514 
1515 			if (strcmp(member_name, name) == 0) {
1516 				if (btf_member_bitfield_size(parent_type, i) != 0) {
1517 					fprintf(stderr, "Bitfield presets are not supported %s\n",
1518 						name);
1519 					return -EINVAL;
1520 				}
1521 				*member_offset = parent_offset + member->offset;
1522 				*member_tid = tid;
1523 				return 0;
1524 			}
1525 		} else if (btf_is_composite(member_type)) {
1526 			int err;
1527 
1528 			err = btf_find_member(btf, member_type, parent_offset + member->offset,
1529 					      member_name, member_tid, member_offset);
1530 			if (!err)
1531 				return 0;
1532 		}
1533 	}
1534 
1535 	return -EINVAL;
1536 }
1537 
1538 static int adjust_var_secinfo(struct btf *btf, const struct btf_type *t,
1539 			      struct btf_var_secinfo *sinfo, const char *var)
1540 {
1541 	char expr[256], *saveptr;
1542 	const struct btf_type *base_type, *member_type;
1543 	int err, member_tid;
1544 	char *name;
1545 	__u32 member_offset = 0;
1546 
1547 	base_type = btf__type_by_id(btf, btf__resolve_type(btf, t->type));
1548 	snprintf(expr, sizeof(expr), "%s", var);
1549 	strtok_r(expr, ".", &saveptr);
1550 
1551 	while ((name = strtok_r(NULL, ".", &saveptr))) {
1552 		err = btf_find_member(btf, base_type, 0, name, &member_tid, &member_offset);
1553 		if (err) {
1554 			fprintf(stderr, "Could not find member %s for variable %s\n", name, var);
1555 			return err;
1556 		}
1557 		member_type = btf__type_by_id(btf, member_tid);
1558 		sinfo->offset += member_offset / 8;
1559 		sinfo->size = member_type->size;
1560 		sinfo->type = member_tid;
1561 		base_type = member_type;
1562 	}
1563 	return 0;
1564 }
1565 
1566 static int set_global_var(struct bpf_object *obj, struct btf *btf,
1567 			  struct bpf_map *map, struct btf_var_secinfo *sinfo,
1568 			  struct var_preset *preset)
1569 {
1570 	const struct btf_type *base_type;
1571 	void *ptr;
1572 	long long value = preset->ivalue;
1573 	size_t size;
1574 
1575 	base_type = btf__type_by_id(btf, btf__resolve_type(btf, sinfo->type));
1576 	if (!base_type) {
1577 		fprintf(stderr, "Failed to resolve type %d\n", sinfo->type);
1578 		return -EINVAL;
1579 	}
1580 	if (!is_preset_supported(base_type)) {
1581 		fprintf(stderr, "Setting value for type %s is not supported\n",
1582 			btf__name_by_offset(btf, base_type->name_off));
1583 		return -EINVAL;
1584 	}
1585 
1586 	if (preset->type == ENUMERATOR) {
1587 		if (btf_is_any_enum(base_type)) {
1588 			if (enum_value_from_name(btf, base_type, preset->svalue, &value)) {
1589 				fprintf(stderr,
1590 					"Failed to find integer value for enum element %s\n",
1591 					preset->svalue);
1592 				return -EINVAL;
1593 			}
1594 		} else {
1595 			fprintf(stderr, "Value %s is not supported for type %s\n",
1596 				preset->svalue, btf__name_by_offset(btf, base_type->name_off));
1597 			return -EINVAL;
1598 		}
1599 	}
1600 
1601 	/* Check if value fits into the target variable size */
1602 	if  (sinfo->size < sizeof(value)) {
1603 		bool is_signed = is_signed_type(base_type);
1604 		__u32 unsigned_bits = sinfo->size * 8 - (is_signed ? 1 : 0);
1605 		long long max_val = 1ll << unsigned_bits;
1606 
1607 		if (value >= max_val || value < -max_val) {
1608 			fprintf(stderr,
1609 				"Variable %s value %lld is out of range [%lld; %lld]\n",
1610 				btf__name_by_offset(btf, base_type->name_off), value,
1611 				is_signed ? -max_val : 0, max_val - 1);
1612 			return -EINVAL;
1613 		}
1614 	}
1615 
1616 	ptr = bpf_map__initial_value(map, &size);
1617 	if (!ptr || sinfo->offset + sinfo->size > size)
1618 		return -EINVAL;
1619 
1620 	if (__BYTE_ORDER == __LITTLE_ENDIAN) {
1621 		memcpy(ptr + sinfo->offset, &value, sinfo->size);
1622 	} else { /* __BYTE_ORDER == __BIG_ENDIAN */
1623 		__u8 src_offset = sizeof(value) - sinfo->size;
1624 
1625 		memcpy(ptr + sinfo->offset, (void *)&value + src_offset, sinfo->size);
1626 	}
1627 	return 0;
1628 }
1629 
1630 static int set_global_vars(struct bpf_object *obj, struct var_preset *presets, int npresets)
1631 {
1632 	struct btf_var_secinfo *sinfo;
1633 	const char *sec_name;
1634 	const struct btf_type *t;
1635 	struct bpf_map *map;
1636 	struct btf *btf;
1637 	int i, j, k, n, cnt, err = 0;
1638 
1639 	if (npresets == 0)
1640 		return 0;
1641 
1642 	btf = bpf_object__btf(obj);
1643 	if (!btf)
1644 		return -EINVAL;
1645 
1646 	cnt = btf__type_cnt(btf);
1647 	for (i = 1; i != cnt; ++i) {
1648 		t = btf__type_by_id(btf, i);
1649 
1650 		if (!btf_is_datasec(t))
1651 			continue;
1652 
1653 		sinfo = btf_var_secinfos(t);
1654 		sec_name = btf__name_by_offset(btf, t->name_off);
1655 		map = bpf_object__find_map_by_name(obj, sec_name);
1656 		if (!map)
1657 			continue;
1658 
1659 		n = btf_vlen(t);
1660 		for (j = 0; j < n; ++j, ++sinfo) {
1661 			const struct btf_type *var_type = btf__type_by_id(btf, sinfo->type);
1662 			const char *var_name;
1663 			int var_len;
1664 
1665 			if (!btf_is_var(var_type))
1666 				continue;
1667 
1668 			var_name = btf__name_by_offset(btf, var_type->name_off);
1669 			var_len = strlen(var_name);
1670 
1671 			for (k = 0; k < npresets; ++k) {
1672 				struct btf_var_secinfo tmp_sinfo;
1673 
1674 				if (strncmp(var_name, presets[k].name, var_len) != 0 ||
1675 				    (presets[k].name[var_len] != '\0' &&
1676 				     presets[k].name[var_len] != '.'))
1677 					continue;
1678 
1679 				if (presets[k].applied) {
1680 					fprintf(stderr, "Variable %s is set more than once",
1681 						var_name);
1682 					return -EINVAL;
1683 				}
1684 				tmp_sinfo = *sinfo;
1685 				err = adjust_var_secinfo(btf, var_type,
1686 							 &tmp_sinfo, presets[k].name);
1687 				if (err)
1688 					return err;
1689 
1690 				err = set_global_var(obj, btf, map, &tmp_sinfo, presets + k);
1691 				if (err)
1692 					return err;
1693 
1694 				presets[k].applied = true;
1695 			}
1696 		}
1697 	}
1698 	for (i = 0; i < npresets; ++i) {
1699 		if (!presets[i].applied) {
1700 			fprintf(stderr, "Global variable preset %s has not been applied\n",
1701 				presets[i].name);
1702 		}
1703 		presets[i].applied = false;
1704 	}
1705 	return err;
1706 }
1707 
1708 static int process_obj(const char *filename)
1709 {
1710 	const char *base_filename = basename(strdupa(filename));
1711 	struct bpf_object *obj = NULL, *tobj;
1712 	struct bpf_program *prog, *tprog, *lprog;
1713 	libbpf_print_fn_t old_libbpf_print_fn;
1714 	LIBBPF_OPTS(bpf_object_open_opts, opts);
1715 	int err = 0, prog_cnt = 0;
1716 
1717 	if (!should_process_file_prog(base_filename, NULL)) {
1718 		if (env.verbose)
1719 			printf("Skipping '%s' due to filters...\n", filename);
1720 		env.files_skipped++;
1721 		return 0;
1722 	}
1723 	if (!is_bpf_obj_file(filename)) {
1724 		if (env.verbose)
1725 			printf("Skipping '%s' as it's not a BPF object file...\n", filename);
1726 		env.files_skipped++;
1727 		return 0;
1728 	}
1729 
1730 	if (!env.quiet && env.out_fmt == RESFMT_TABLE)
1731 		printf("Processing '%s'...\n", base_filename);
1732 
1733 	old_libbpf_print_fn = libbpf_set_print(libbpf_print_fn);
1734 	obj = bpf_object__open_file(filename, &opts);
1735 	if (!obj) {
1736 		/* if libbpf can't open BPF object file, it could be because
1737 		 * that BPF object file is incomplete and has to be statically
1738 		 * linked into a final BPF object file; instead of bailing
1739 		 * out, report it into stderr, mark it as skipped, and
1740 		 * proceed
1741 		 */
1742 		fprintf(stderr, "Failed to open '%s': %d\n", filename, -errno);
1743 		env.files_skipped++;
1744 		err = 0;
1745 		goto cleanup;
1746 	}
1747 
1748 	env.files_processed++;
1749 
1750 	bpf_object__for_each_program(prog, obj) {
1751 		prog_cnt++;
1752 	}
1753 
1754 	if (prog_cnt == 1) {
1755 		prog = bpf_object__next_program(obj, NULL);
1756 		bpf_program__set_autoload(prog, true);
1757 		err = set_global_vars(obj, env.presets, env.npresets);
1758 		if (err) {
1759 			fprintf(stderr, "Failed to set global variables %d\n", err);
1760 			goto cleanup;
1761 		}
1762 		process_prog(filename, obj, prog);
1763 		goto cleanup;
1764 	}
1765 
1766 	bpf_object__for_each_program(prog, obj) {
1767 		const char *prog_name = bpf_program__name(prog);
1768 
1769 		tobj = bpf_object__open_file(filename, &opts);
1770 		if (!tobj) {
1771 			err = -errno;
1772 			fprintf(stderr, "Failed to open '%s': %d\n", filename, err);
1773 			goto cleanup;
1774 		}
1775 
1776 		err = set_global_vars(tobj, env.presets, env.npresets);
1777 		if (err) {
1778 			fprintf(stderr, "Failed to set global variables %d\n", err);
1779 			goto cleanup;
1780 		}
1781 
1782 		lprog = NULL;
1783 		bpf_object__for_each_program(tprog, tobj) {
1784 			const char *tprog_name = bpf_program__name(tprog);
1785 
1786 			if (strcmp(prog_name, tprog_name) == 0) {
1787 				bpf_program__set_autoload(tprog, true);
1788 				lprog = tprog;
1789 			} else {
1790 				bpf_program__set_autoload(tprog, false);
1791 			}
1792 		}
1793 
1794 		process_prog(filename, tobj, lprog);
1795 		bpf_object__close(tobj);
1796 	}
1797 
1798 cleanup:
1799 	bpf_object__close(obj);
1800 	libbpf_set_print(old_libbpf_print_fn);
1801 	return err;
1802 }
1803 
1804 static int cmp_stat(const struct verif_stats *s1, const struct verif_stats *s2,
1805 		    enum stat_id id, bool asc, bool abs)
1806 {
1807 	int cmp = 0;
1808 
1809 	switch (id) {
1810 	case FILE_NAME:
1811 		cmp = strcmp(s1->file_name, s2->file_name);
1812 		break;
1813 	case PROG_NAME:
1814 		cmp = strcmp(s1->prog_name, s2->prog_name);
1815 		break;
1816 	case ATTACH_TYPE:
1817 	case PROG_TYPE:
1818 	case SIZE:
1819 	case JITED_SIZE:
1820 	case STACK:
1821 	case VERDICT:
1822 	case DURATION:
1823 	case TOTAL_INSNS:
1824 	case TOTAL_STATES:
1825 	case PEAK_STATES:
1826 	case MAX_STATES_PER_INSN:
1827 	case MARK_READ_MAX_LEN: {
1828 		long v1 = s1->stats[id];
1829 		long v2 = s2->stats[id];
1830 
1831 		if (abs) {
1832 			v1 = v1 < 0 ? -v1 : v1;
1833 			v2 = v2 < 0 ? -v2 : v2;
1834 		}
1835 
1836 		if (v1 != v2)
1837 			cmp = v1 < v2 ? -1 : 1;
1838 		break;
1839 	}
1840 	default:
1841 		fprintf(stderr, "Unrecognized stat #%d\n", id);
1842 		exit(1);
1843 	}
1844 
1845 	return asc ? cmp : -cmp;
1846 }
1847 
1848 static int cmp_prog_stats(const void *v1, const void *v2)
1849 {
1850 	const struct verif_stats *s1 = v1, *s2 = v2;
1851 	int i, cmp;
1852 
1853 	for (i = 0; i < env.sort_spec.spec_cnt; i++) {
1854 		cmp = cmp_stat(s1, s2, env.sort_spec.ids[i],
1855 			       env.sort_spec.asc[i], env.sort_spec.abs[i]);
1856 		if (cmp != 0)
1857 			return cmp;
1858 	}
1859 
1860 	/* always disambiguate with file+prog, which are unique */
1861 	cmp = strcmp(s1->file_name, s2->file_name);
1862 	if (cmp != 0)
1863 		return cmp;
1864 	return strcmp(s1->prog_name, s2->prog_name);
1865 }
1866 
1867 static void fetch_join_stat_value(const struct verif_stats_join *s,
1868 				  enum stat_id id, enum stat_variant var,
1869 				  const char **str_val,
1870 				  double *num_val)
1871 {
1872 	long v1, v2;
1873 
1874 	if (id == FILE_NAME) {
1875 		*str_val = s->file_name;
1876 		return;
1877 	}
1878 	if (id == PROG_NAME) {
1879 		*str_val = s->prog_name;
1880 		return;
1881 	}
1882 
1883 	v1 = s->stats_a ? s->stats_a->stats[id] : 0;
1884 	v2 = s->stats_b ? s->stats_b->stats[id] : 0;
1885 
1886 	switch (var) {
1887 	case VARIANT_A:
1888 		if (!s->stats_a)
1889 			*num_val = -DBL_MAX;
1890 		else
1891 			*num_val = s->stats_a->stats[id];
1892 		return;
1893 	case VARIANT_B:
1894 		if (!s->stats_b)
1895 			*num_val = -DBL_MAX;
1896 		else
1897 			*num_val = s->stats_b->stats[id];
1898 		return;
1899 	case VARIANT_DIFF:
1900 		if (!s->stats_a || !s->stats_b)
1901 			*num_val = -DBL_MAX;
1902 		else if (id == VERDICT)
1903 			*num_val = v1 == v2 ? 1.0 /* MATCH */ : 0.0 /* MISMATCH */;
1904 		else
1905 			*num_val = (double)(v2 - v1);
1906 		return;
1907 	case VARIANT_PCT:
1908 		if (!s->stats_a || !s->stats_b) {
1909 			*num_val = -DBL_MAX;
1910 		} else if (v1 == 0) {
1911 			if (v1 == v2)
1912 				*num_val = 0.0;
1913 			else
1914 				*num_val = v2 < v1 ? -100.0 : 100.0;
1915 		} else {
1916 			 *num_val = (v2 - v1) * 100.0 / v1;
1917 		}
1918 		return;
1919 	}
1920 }
1921 
1922 static int cmp_join_stat(const struct verif_stats_join *s1,
1923 			 const struct verif_stats_join *s2,
1924 			 enum stat_id id, enum stat_variant var,
1925 			 bool asc, bool abs)
1926 {
1927 	const char *str1 = NULL, *str2 = NULL;
1928 	double v1 = 0.0, v2 = 0.0;
1929 	int cmp = 0;
1930 
1931 	fetch_join_stat_value(s1, id, var, &str1, &v1);
1932 	fetch_join_stat_value(s2, id, var, &str2, &v2);
1933 
1934 	if (abs) {
1935 		v1 = fabs(v1);
1936 		v2 = fabs(v2);
1937 	}
1938 
1939 	if (str1)
1940 		cmp = strcmp(str1, str2);
1941 	else if (v1 != v2)
1942 		cmp = v1 < v2 ? -1 : 1;
1943 
1944 	return asc ? cmp : -cmp;
1945 }
1946 
1947 static int cmp_join_stats(const void *v1, const void *v2)
1948 {
1949 	const struct verif_stats_join *s1 = v1, *s2 = v2;
1950 	int i, cmp;
1951 
1952 	for (i = 0; i < env.sort_spec.spec_cnt; i++) {
1953 		cmp = cmp_join_stat(s1, s2,
1954 				    env.sort_spec.ids[i],
1955 				    env.sort_spec.variants[i],
1956 				    env.sort_spec.asc[i],
1957 				    env.sort_spec.abs[i]);
1958 		if (cmp != 0)
1959 			return cmp;
1960 	}
1961 
1962 	/* always disambiguate with file+prog, which are unique */
1963 	cmp = strcmp(s1->file_name, s2->file_name);
1964 	if (cmp != 0)
1965 		return cmp;
1966 	return strcmp(s1->prog_name, s2->prog_name);
1967 }
1968 
1969 #define HEADER_CHAR '-'
1970 #define COLUMN_SEP "  "
1971 
1972 static void output_header_underlines(void)
1973 {
1974 	int i, j, len;
1975 
1976 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
1977 		len = env.output_spec.lens[i];
1978 
1979 		printf("%s", i == 0 ? "" : COLUMN_SEP);
1980 		for (j = 0; j < len; j++)
1981 			printf("%c", HEADER_CHAR);
1982 	}
1983 	printf("\n");
1984 }
1985 
1986 static void output_headers(enum resfmt fmt)
1987 {
1988 	const char *fmt_str;
1989 	int i, len;
1990 
1991 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
1992 		int id = env.output_spec.ids[i];
1993 		int *max_len = &env.output_spec.lens[i];
1994 
1995 		switch (fmt) {
1996 		case RESFMT_TABLE_CALCLEN:
1997 			len = snprintf(NULL, 0, "%s", stat_defs[id].header);
1998 			if (len > *max_len)
1999 				*max_len = len;
2000 			break;
2001 		case RESFMT_TABLE:
2002 			fmt_str = stat_defs[id].left_aligned ? "%s%-*s" : "%s%*s";
2003 			printf(fmt_str, i == 0 ? "" : COLUMN_SEP,  *max_len, stat_defs[id].header);
2004 			if (i == env.output_spec.spec_cnt - 1)
2005 				printf("\n");
2006 			break;
2007 		case RESFMT_CSV:
2008 			printf("%s%s", i == 0 ? "" : ",", stat_defs[id].names[0]);
2009 			if (i == env.output_spec.spec_cnt - 1)
2010 				printf("\n");
2011 			break;
2012 		}
2013 	}
2014 
2015 	if (fmt == RESFMT_TABLE)
2016 		output_header_underlines();
2017 }
2018 
2019 static void prepare_value(const struct verif_stats *s, enum stat_id id,
2020 			  const char **str, long *val)
2021 {
2022 	switch (id) {
2023 	case FILE_NAME:
2024 		*str = s ? s->file_name : "N/A";
2025 		break;
2026 	case PROG_NAME:
2027 		*str = s ? s->prog_name : "N/A";
2028 		break;
2029 	case VERDICT:
2030 		if (!s)
2031 			*str = "N/A";
2032 		else
2033 			*str = s->stats[VERDICT] ? "success" : "failure";
2034 		break;
2035 	case ATTACH_TYPE:
2036 		if (!s)
2037 			*str = "N/A";
2038 		else
2039 			*str = libbpf_bpf_attach_type_str(s->stats[ATTACH_TYPE]) ?: "N/A";
2040 		break;
2041 	case PROG_TYPE:
2042 		if (!s)
2043 			*str = "N/A";
2044 		else
2045 			*str = libbpf_bpf_prog_type_str(s->stats[PROG_TYPE]) ?: "N/A";
2046 		break;
2047 	case DURATION:
2048 	case TOTAL_INSNS:
2049 	case TOTAL_STATES:
2050 	case PEAK_STATES:
2051 	case MAX_STATES_PER_INSN:
2052 	case MARK_READ_MAX_LEN:
2053 	case STACK:
2054 	case SIZE:
2055 	case JITED_SIZE:
2056 		*val = s ? s->stats[id] : 0;
2057 		break;
2058 	default:
2059 		fprintf(stderr, "Unrecognized stat #%d\n", id);
2060 		exit(1);
2061 	}
2062 }
2063 
2064 static void output_stats(const struct verif_stats *s, enum resfmt fmt, bool last)
2065 {
2066 	int i;
2067 
2068 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
2069 		int id = env.output_spec.ids[i];
2070 		int *max_len = &env.output_spec.lens[i], len;
2071 		const char *str = NULL;
2072 		long val = 0;
2073 
2074 		prepare_value(s, id, &str, &val);
2075 
2076 		switch (fmt) {
2077 		case RESFMT_TABLE_CALCLEN:
2078 			if (str)
2079 				len = snprintf(NULL, 0, "%s", str);
2080 			else
2081 				len = snprintf(NULL, 0, "%ld", val);
2082 			if (len > *max_len)
2083 				*max_len = len;
2084 			break;
2085 		case RESFMT_TABLE:
2086 			if (str)
2087 				printf("%s%-*s", i == 0 ? "" : COLUMN_SEP, *max_len, str);
2088 			else
2089 				printf("%s%*ld", i == 0 ? "" : COLUMN_SEP,  *max_len, val);
2090 			if (i == env.output_spec.spec_cnt - 1)
2091 				printf("\n");
2092 			break;
2093 		case RESFMT_CSV:
2094 			if (str)
2095 				printf("%s%s", i == 0 ? "" : ",", str);
2096 			else
2097 				printf("%s%ld", i == 0 ? "" : ",", val);
2098 			if (i == env.output_spec.spec_cnt - 1)
2099 				printf("\n");
2100 			break;
2101 		}
2102 	}
2103 
2104 	if (last && fmt == RESFMT_TABLE) {
2105 		output_header_underlines();
2106 		printf("Done. Processed %d files, %d programs. Skipped %d files, %d programs.\n",
2107 		       env.files_processed, env.files_skipped, env.progs_processed, env.progs_skipped);
2108 	}
2109 }
2110 
2111 static int parse_stat_value(const char *str, enum stat_id id, struct verif_stats *st)
2112 {
2113 	switch (id) {
2114 	case FILE_NAME:
2115 		st->file_name = strdup(str);
2116 		if (!st->file_name)
2117 			return -ENOMEM;
2118 		break;
2119 	case PROG_NAME:
2120 		st->prog_name = strdup(str);
2121 		if (!st->prog_name)
2122 			return -ENOMEM;
2123 		break;
2124 	case VERDICT:
2125 		if (strcmp(str, "success") == 0) {
2126 			st->stats[VERDICT] = true;
2127 		} else if (strcmp(str, "failure") == 0) {
2128 			st->stats[VERDICT] = false;
2129 		} else {
2130 			fprintf(stderr, "Unrecognized verification verdict '%s'\n", str);
2131 			return -EINVAL;
2132 		}
2133 		break;
2134 	case DURATION:
2135 	case TOTAL_INSNS:
2136 	case TOTAL_STATES:
2137 	case PEAK_STATES:
2138 	case MAX_STATES_PER_INSN:
2139 	case MARK_READ_MAX_LEN:
2140 	case SIZE:
2141 	case JITED_SIZE:
2142 	case STACK: {
2143 		long val;
2144 		int err, n;
2145 
2146 		if (sscanf(str, "%ld %n", &val, &n) != 1 || n != strlen(str)) {
2147 			err = -errno;
2148 			fprintf(stderr, "Failed to parse '%s' as integer\n", str);
2149 			return err;
2150 		}
2151 
2152 		st->stats[id] = val;
2153 		break;
2154 	}
2155 	case PROG_TYPE: {
2156 		enum bpf_prog_type prog_type = 0;
2157 		const char *type;
2158 
2159 		while ((type = libbpf_bpf_prog_type_str(prog_type)))  {
2160 			if (strcmp(type, str) == 0) {
2161 				st->stats[id] = prog_type;
2162 				break;
2163 			}
2164 			prog_type++;
2165 		}
2166 
2167 		if (!type) {
2168 			fprintf(stderr, "Unrecognized prog type %s\n", str);
2169 			return -EINVAL;
2170 		}
2171 		break;
2172 	}
2173 	case ATTACH_TYPE: {
2174 		enum bpf_attach_type attach_type = 0;
2175 		const char *type;
2176 
2177 		while ((type = libbpf_bpf_attach_type_str(attach_type)))  {
2178 			if (strcmp(type, str) == 0) {
2179 				st->stats[id] = attach_type;
2180 				break;
2181 			}
2182 			attach_type++;
2183 		}
2184 
2185 		if (!type) {
2186 			fprintf(stderr, "Unrecognized attach type %s\n", str);
2187 			return -EINVAL;
2188 		}
2189 		break;
2190 	}
2191 	default:
2192 		fprintf(stderr, "Unrecognized stat #%d\n", id);
2193 		return -EINVAL;
2194 	}
2195 	return 0;
2196 }
2197 
2198 static int parse_stats_csv(const char *filename, struct stat_specs *specs,
2199 			   struct verif_stats **statsp, int *stat_cntp)
2200 {
2201 	char line[4096];
2202 	FILE *f;
2203 	int err = 0;
2204 	bool header = true;
2205 
2206 	f = fopen(filename, "r");
2207 	if (!f) {
2208 		err = -errno;
2209 		fprintf(stderr, "Failed to open '%s': %d\n", filename, err);
2210 		return err;
2211 	}
2212 
2213 	*stat_cntp = 0;
2214 
2215 	while (fgets(line, sizeof(line), f)) {
2216 		char *input = line, *state = NULL, *next;
2217 		struct verif_stats *st = NULL;
2218 		int col = 0, cnt = 0;
2219 
2220 		if (!header) {
2221 			void *tmp;
2222 
2223 			tmp = realloc(*statsp, (*stat_cntp + 1) * sizeof(**statsp));
2224 			if (!tmp) {
2225 				err = -ENOMEM;
2226 				goto cleanup;
2227 			}
2228 			*statsp = tmp;
2229 
2230 			st = &(*statsp)[*stat_cntp];
2231 			memset(st, 0, sizeof(*st));
2232 
2233 			*stat_cntp += 1;
2234 		}
2235 
2236 		while ((next = strtok_r(cnt++ ? NULL : input, ",\n", &state))) {
2237 			if (header) {
2238 				/* for the first line, set up spec stats */
2239 				err = parse_stat(next, specs);
2240 				if (err)
2241 					goto cleanup;
2242 				continue;
2243 			}
2244 
2245 			/* for all other lines, parse values based on spec */
2246 			if (col >= specs->spec_cnt) {
2247 				fprintf(stderr, "Found extraneous column #%d in row #%d of '%s'\n",
2248 					col, *stat_cntp, filename);
2249 				err = -EINVAL;
2250 				goto cleanup;
2251 			}
2252 			err = parse_stat_value(next, specs->ids[col], st);
2253 			if (err)
2254 				goto cleanup;
2255 			col++;
2256 		}
2257 
2258 		if (header) {
2259 			header = false;
2260 			continue;
2261 		}
2262 
2263 		if (col < specs->spec_cnt) {
2264 			fprintf(stderr, "Not enough columns in row #%d in '%s'\n",
2265 				*stat_cntp, filename);
2266 			err = -EINVAL;
2267 			goto cleanup;
2268 		}
2269 
2270 		if (!st->file_name || !st->prog_name) {
2271 			fprintf(stderr, "Row #%d in '%s' is missing file and/or program name\n",
2272 				*stat_cntp, filename);
2273 			err = -EINVAL;
2274 			goto cleanup;
2275 		}
2276 
2277 		/* in comparison mode we can only check filters after we
2278 		 * parsed entire line; if row should be ignored we pretend we
2279 		 * never parsed it
2280 		 */
2281 		if (!should_process_file_prog(st->file_name, st->prog_name)) {
2282 			free(st->file_name);
2283 			free(st->prog_name);
2284 			*stat_cntp -= 1;
2285 		}
2286 	}
2287 
2288 	if (!feof(f)) {
2289 		err = -errno;
2290 		fprintf(stderr, "Failed I/O for '%s': %d\n", filename, err);
2291 	}
2292 
2293 cleanup:
2294 	fclose(f);
2295 	return err;
2296 }
2297 
2298 /* empty/zero stats for mismatched rows */
2299 static const struct verif_stats fallback_stats = { .file_name = "", .prog_name = "" };
2300 
2301 static bool is_key_stat(enum stat_id id)
2302 {
2303 	return id == FILE_NAME || id == PROG_NAME;
2304 }
2305 
2306 static void output_comp_header_underlines(void)
2307 {
2308 	int i, j, k;
2309 
2310 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
2311 		int id = env.output_spec.ids[i];
2312 		int max_j = is_key_stat(id) ? 1 : 3;
2313 
2314 		for (j = 0; j < max_j; j++) {
2315 			int len = env.output_spec.lens[3 * i + j];
2316 
2317 			printf("%s", i + j == 0 ? "" : COLUMN_SEP);
2318 
2319 			for (k = 0; k < len; k++)
2320 				printf("%c", HEADER_CHAR);
2321 		}
2322 	}
2323 	printf("\n");
2324 }
2325 
2326 static void output_comp_headers(enum resfmt fmt)
2327 {
2328 	static const char *table_sfxs[3] = {" (A)", " (B)", " (DIFF)"};
2329 	static const char *name_sfxs[3] = {"_base", "_comp", "_diff"};
2330 	int i, j, len;
2331 
2332 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
2333 		int id = env.output_spec.ids[i];
2334 		/* key stats don't have A/B/DIFF columns, they are common for both data sets */
2335 		int max_j = is_key_stat(id) ? 1 : 3;
2336 
2337 		for (j = 0; j < max_j; j++) {
2338 			int *max_len = &env.output_spec.lens[3 * i + j];
2339 			bool last = (i == env.output_spec.spec_cnt - 1) && (j == max_j - 1);
2340 			const char *sfx;
2341 
2342 			switch (fmt) {
2343 			case RESFMT_TABLE_CALCLEN:
2344 				sfx = is_key_stat(id) ? "" : table_sfxs[j];
2345 				len = snprintf(NULL, 0, "%s%s", stat_defs[id].header, sfx);
2346 				if (len > *max_len)
2347 					*max_len = len;
2348 				break;
2349 			case RESFMT_TABLE:
2350 				sfx = is_key_stat(id) ? "" : table_sfxs[j];
2351 				printf("%s%-*s%s", i + j == 0 ? "" : COLUMN_SEP,
2352 				       *max_len - (int)strlen(sfx), stat_defs[id].header, sfx);
2353 				if (last)
2354 					printf("\n");
2355 				break;
2356 			case RESFMT_CSV:
2357 				sfx = is_key_stat(id) ? "" : name_sfxs[j];
2358 				printf("%s%s%s", i + j == 0 ? "" : ",", stat_defs[id].names[0], sfx);
2359 				if (last)
2360 					printf("\n");
2361 				break;
2362 			}
2363 		}
2364 	}
2365 
2366 	if (fmt == RESFMT_TABLE)
2367 		output_comp_header_underlines();
2368 }
2369 
2370 static void output_comp_stats(const struct verif_stats_join *join_stats,
2371 			      enum resfmt fmt, bool last)
2372 {
2373 	const struct verif_stats *base = join_stats->stats_a;
2374 	const struct verif_stats *comp = join_stats->stats_b;
2375 	char base_buf[1024] = {}, comp_buf[1024] = {}, diff_buf[1024] = {};
2376 	int i;
2377 
2378 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
2379 		int id = env.output_spec.ids[i], len;
2380 		int *max_len_base = &env.output_spec.lens[3 * i + 0];
2381 		int *max_len_comp = &env.output_spec.lens[3 * i + 1];
2382 		int *max_len_diff = &env.output_spec.lens[3 * i + 2];
2383 		const char *base_str = NULL, *comp_str = NULL;
2384 		long base_val = 0, comp_val = 0, diff_val = 0;
2385 
2386 		prepare_value(base, id, &base_str, &base_val);
2387 		prepare_value(comp, id, &comp_str, &comp_val);
2388 
2389 		/* normalize all the outputs to be in string buffers for simplicity */
2390 		if (is_key_stat(id)) {
2391 			/* key stats (file and program name) are always strings */
2392 			if (base)
2393 				snprintf(base_buf, sizeof(base_buf), "%s", base_str);
2394 			else
2395 				snprintf(base_buf, sizeof(base_buf), "%s", comp_str);
2396 		} else if (base_str) {
2397 			snprintf(base_buf, sizeof(base_buf), "%s", base_str);
2398 			snprintf(comp_buf, sizeof(comp_buf), "%s", comp_str);
2399 			if (!base || !comp)
2400 				snprintf(diff_buf, sizeof(diff_buf), "%s", "N/A");
2401 			else if (strcmp(base_str, comp_str) == 0)
2402 				snprintf(diff_buf, sizeof(diff_buf), "%s", "MATCH");
2403 			else
2404 				snprintf(diff_buf, sizeof(diff_buf), "%s", "MISMATCH");
2405 		} else {
2406 			double p = 0.0;
2407 
2408 			if (base)
2409 				snprintf(base_buf, sizeof(base_buf), "%ld", base_val);
2410 			else
2411 				snprintf(base_buf, sizeof(base_buf), "%s", "N/A");
2412 			if (comp)
2413 				snprintf(comp_buf, sizeof(comp_buf), "%ld", comp_val);
2414 			else
2415 				snprintf(comp_buf, sizeof(comp_buf), "%s", "N/A");
2416 
2417 			diff_val = comp_val - base_val;
2418 			if (!base || !comp) {
2419 				snprintf(diff_buf, sizeof(diff_buf), "%s", "N/A");
2420 			} else {
2421 				if (base_val == 0) {
2422 					if (comp_val == base_val)
2423 						p = 0.0; /* avoid +0 (+100%) case */
2424 					else
2425 						p = comp_val < base_val ? -100.0 : 100.0;
2426 				} else {
2427 					 p = diff_val * 100.0 / base_val;
2428 				}
2429 				snprintf(diff_buf, sizeof(diff_buf), "%+ld (%+.2lf%%)", diff_val, p);
2430 			}
2431 		}
2432 
2433 		switch (fmt) {
2434 		case RESFMT_TABLE_CALCLEN:
2435 			len = strlen(base_buf);
2436 			if (len > *max_len_base)
2437 				*max_len_base = len;
2438 			if (!is_key_stat(id)) {
2439 				len = strlen(comp_buf);
2440 				if (len > *max_len_comp)
2441 					*max_len_comp = len;
2442 				len = strlen(diff_buf);
2443 				if (len > *max_len_diff)
2444 					*max_len_diff = len;
2445 			}
2446 			break;
2447 		case RESFMT_TABLE: {
2448 			/* string outputs are left-aligned, number outputs are right-aligned */
2449 			const char *fmt = base_str ? "%s%-*s" : "%s%*s";
2450 
2451 			printf(fmt, i == 0 ? "" : COLUMN_SEP, *max_len_base, base_buf);
2452 			if (!is_key_stat(id)) {
2453 				printf(fmt, COLUMN_SEP, *max_len_comp, comp_buf);
2454 				printf(fmt, COLUMN_SEP, *max_len_diff, diff_buf);
2455 			}
2456 			if (i == env.output_spec.spec_cnt - 1)
2457 				printf("\n");
2458 			break;
2459 		}
2460 		case RESFMT_CSV:
2461 			printf("%s%s", i == 0 ? "" : ",", base_buf);
2462 			if (!is_key_stat(id)) {
2463 				printf("%s%s", i == 0 ? "" : ",", comp_buf);
2464 				printf("%s%s", i == 0 ? "" : ",", diff_buf);
2465 			}
2466 			if (i == env.output_spec.spec_cnt - 1)
2467 				printf("\n");
2468 			break;
2469 		}
2470 	}
2471 
2472 	if (last && fmt == RESFMT_TABLE)
2473 		output_comp_header_underlines();
2474 }
2475 
2476 static int cmp_stats_key(const struct verif_stats *base, const struct verif_stats *comp)
2477 {
2478 	int r;
2479 
2480 	r = strcmp(base->file_name, comp->file_name);
2481 	if (r != 0)
2482 		return r;
2483 	return strcmp(base->prog_name, comp->prog_name);
2484 }
2485 
2486 static bool is_join_stat_filter_matched(struct filter *f, const struct verif_stats_join *stats)
2487 {
2488 	static const double eps = 1e-9;
2489 	const char *str = NULL;
2490 	double value = 0.0;
2491 
2492 	fetch_join_stat_value(stats, f->stat_id, f->stat_var, &str, &value);
2493 
2494 	if (f->abs)
2495 		value = fabs(value);
2496 
2497 	switch (f->op) {
2498 	case OP_EQ: return value > f->value - eps && value < f->value + eps;
2499 	case OP_NEQ: return value < f->value - eps || value > f->value + eps;
2500 	case OP_LT: return value < f->value - eps;
2501 	case OP_LE: return value <= f->value + eps;
2502 	case OP_GT: return value > f->value + eps;
2503 	case OP_GE: return value >= f->value - eps;
2504 	}
2505 
2506 	fprintf(stderr, "BUG: unknown filter op %d!\n", f->op);
2507 	return false;
2508 }
2509 
2510 static bool should_output_join_stats(const struct verif_stats_join *stats)
2511 {
2512 	struct filter *f;
2513 	int i, allow_cnt = 0;
2514 
2515 	for (i = 0; i < env.deny_filter_cnt; i++) {
2516 		f = &env.deny_filters[i];
2517 		if (f->kind != FILTER_STAT)
2518 			continue;
2519 
2520 		if (is_join_stat_filter_matched(f, stats))
2521 			return false;
2522 	}
2523 
2524 	for (i = 0; i < env.allow_filter_cnt; i++) {
2525 		f = &env.allow_filters[i];
2526 		if (f->kind != FILTER_STAT)
2527 			continue;
2528 		allow_cnt++;
2529 
2530 		if (is_join_stat_filter_matched(f, stats))
2531 			return true;
2532 	}
2533 
2534 	/* if there are no stat allowed filters, pass everything through */
2535 	return allow_cnt == 0;
2536 }
2537 
2538 static int handle_comparison_mode(void)
2539 {
2540 	struct stat_specs base_specs = {}, comp_specs = {};
2541 	struct stat_specs tmp_sort_spec;
2542 	enum resfmt cur_fmt;
2543 	int err, i, j, last_idx, cnt;
2544 
2545 	if (env.filename_cnt != 2) {
2546 		fprintf(stderr, "Comparison mode expects exactly two input CSV files!\n\n");
2547 		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
2548 		return -EINVAL;
2549 	}
2550 
2551 	err = parse_stats_csv(env.filenames[0], &base_specs,
2552 			      &env.baseline_stats, &env.baseline_stat_cnt);
2553 	if (err) {
2554 		fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err);
2555 		return err;
2556 	}
2557 	err = parse_stats_csv(env.filenames[1], &comp_specs,
2558 			      &env.prog_stats, &env.prog_stat_cnt);
2559 	if (err) {
2560 		fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[1], err);
2561 		return err;
2562 	}
2563 
2564 	/* To keep it simple we validate that the set and order of stats in
2565 	 * both CSVs are exactly the same. This can be lifted with a bit more
2566 	 * pre-processing later.
2567 	 */
2568 	if (base_specs.spec_cnt != comp_specs.spec_cnt) {
2569 		fprintf(stderr, "Number of stats in '%s' and '%s' differs (%d != %d)!\n",
2570 			env.filenames[0], env.filenames[1],
2571 			base_specs.spec_cnt, comp_specs.spec_cnt);
2572 		return -EINVAL;
2573 	}
2574 	for (i = 0; i < base_specs.spec_cnt; i++) {
2575 		if (base_specs.ids[i] != comp_specs.ids[i]) {
2576 			fprintf(stderr, "Stats composition differs between '%s' and '%s' (%s != %s)!\n",
2577 				env.filenames[0], env.filenames[1],
2578 				stat_defs[base_specs.ids[i]].names[0],
2579 				stat_defs[comp_specs.ids[i]].names[0]);
2580 			return -EINVAL;
2581 		}
2582 	}
2583 
2584 	/* Replace user-specified sorting spec with file+prog sorting rule to
2585 	 * be able to join two datasets correctly. Once we are done, we will
2586 	 * restore the original sort spec.
2587 	 */
2588 	tmp_sort_spec = env.sort_spec;
2589 	env.sort_spec = join_sort_spec;
2590 	qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);
2591 	qsort(env.baseline_stats, env.baseline_stat_cnt, sizeof(*env.baseline_stats), cmp_prog_stats);
2592 	env.sort_spec = tmp_sort_spec;
2593 
2594 	/* Join two datasets together. If baseline and comparison datasets
2595 	 * have different subset of rows (we match by 'object + prog' as
2596 	 * a unique key) then assume empty/missing/zero value for rows that
2597 	 * are missing in the opposite data set.
2598 	 */
2599 	i = j = 0;
2600 	while (i < env.baseline_stat_cnt || j < env.prog_stat_cnt) {
2601 		const struct verif_stats *base, *comp;
2602 		struct verif_stats_join *join;
2603 		void *tmp;
2604 		int r;
2605 
2606 		base = i < env.baseline_stat_cnt ? &env.baseline_stats[i] : &fallback_stats;
2607 		comp = j < env.prog_stat_cnt ? &env.prog_stats[j] : &fallback_stats;
2608 
2609 		if (!base->file_name || !base->prog_name) {
2610 			fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n",
2611 				i, env.filenames[0]);
2612 			return -EINVAL;
2613 		}
2614 		if (!comp->file_name || !comp->prog_name) {
2615 			fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n",
2616 				j, env.filenames[1]);
2617 			return -EINVAL;
2618 		}
2619 
2620 		tmp = realloc(env.join_stats, (env.join_stat_cnt + 1) * sizeof(*env.join_stats));
2621 		if (!tmp)
2622 			return -ENOMEM;
2623 		env.join_stats = tmp;
2624 
2625 		join = &env.join_stats[env.join_stat_cnt];
2626 		memset(join, 0, sizeof(*join));
2627 
2628 		r = cmp_stats_key(base, comp);
2629 		if (r == 0) {
2630 			join->file_name = base->file_name;
2631 			join->prog_name = base->prog_name;
2632 			join->stats_a = base;
2633 			join->stats_b = comp;
2634 			i++;
2635 			j++;
2636 		} else if (base != &fallback_stats && (comp == &fallback_stats || r < 0)) {
2637 			join->file_name = base->file_name;
2638 			join->prog_name = base->prog_name;
2639 			join->stats_a = base;
2640 			join->stats_b = NULL;
2641 			i++;
2642 		} else if (comp != &fallback_stats && (base == &fallback_stats || r > 0)) {
2643 			join->file_name = comp->file_name;
2644 			join->prog_name = comp->prog_name;
2645 			join->stats_a = NULL;
2646 			join->stats_b = comp;
2647 			j++;
2648 		} else {
2649 			fprintf(stderr, "%s:%d: should never reach here i=%i, j=%i",
2650 				__FILE__, __LINE__, i, j);
2651 			return -EINVAL;
2652 		}
2653 		env.join_stat_cnt += 1;
2654 	}
2655 
2656 	/* now sort joined results according to sort spec */
2657 	qsort(env.join_stats, env.join_stat_cnt, sizeof(*env.join_stats), cmp_join_stats);
2658 
2659 	/* for human-readable table output we need to do extra pass to
2660 	 * calculate column widths, so we substitute current output format
2661 	 * with RESFMT_TABLE_CALCLEN and later revert it back to RESFMT_TABLE
2662 	 * and do everything again.
2663 	 */
2664 	if (env.out_fmt == RESFMT_TABLE)
2665 		cur_fmt = RESFMT_TABLE_CALCLEN;
2666 	else
2667 		cur_fmt = env.out_fmt;
2668 
2669 one_more_time:
2670 	output_comp_headers(cur_fmt);
2671 
2672 	last_idx = -1;
2673 	cnt = 0;
2674 	for (i = 0; i < env.join_stat_cnt; i++) {
2675 		const struct verif_stats_join *join = &env.join_stats[i];
2676 
2677 		if (!should_output_join_stats(join))
2678 			continue;
2679 
2680 		if (env.top_n && cnt >= env.top_n)
2681 			break;
2682 
2683 		if (cur_fmt == RESFMT_TABLE_CALCLEN)
2684 			last_idx = i;
2685 
2686 		output_comp_stats(join, cur_fmt, i == last_idx);
2687 
2688 		cnt++;
2689 	}
2690 
2691 	if (cur_fmt == RESFMT_TABLE_CALCLEN) {
2692 		cur_fmt = RESFMT_TABLE;
2693 		goto one_more_time; /* ... this time with feeling */
2694 	}
2695 
2696 	return 0;
2697 }
2698 
2699 static bool is_stat_filter_matched(struct filter *f, const struct verif_stats *stats)
2700 {
2701 	long value = stats->stats[f->stat_id];
2702 
2703 	if (f->abs)
2704 		value = value < 0 ? -value : value;
2705 
2706 	switch (f->op) {
2707 	case OP_EQ: return value == f->value;
2708 	case OP_NEQ: return value != f->value;
2709 	case OP_LT: return value < f->value;
2710 	case OP_LE: return value <= f->value;
2711 	case OP_GT: return value > f->value;
2712 	case OP_GE: return value >= f->value;
2713 	}
2714 
2715 	fprintf(stderr, "BUG: unknown filter op %d!\n", f->op);
2716 	return false;
2717 }
2718 
2719 static bool should_output_stats(const struct verif_stats *stats)
2720 {
2721 	struct filter *f;
2722 	int i, allow_cnt = 0;
2723 
2724 	for (i = 0; i < env.deny_filter_cnt; i++) {
2725 		f = &env.deny_filters[i];
2726 		if (f->kind != FILTER_STAT)
2727 			continue;
2728 
2729 		if (is_stat_filter_matched(f, stats))
2730 			return false;
2731 	}
2732 
2733 	for (i = 0; i < env.allow_filter_cnt; i++) {
2734 		f = &env.allow_filters[i];
2735 		if (f->kind != FILTER_STAT)
2736 			continue;
2737 		allow_cnt++;
2738 
2739 		if (is_stat_filter_matched(f, stats))
2740 			return true;
2741 	}
2742 
2743 	/* if there are no stat allowed filters, pass everything through */
2744 	return allow_cnt == 0;
2745 }
2746 
2747 static void output_prog_stats(void)
2748 {
2749 	const struct verif_stats *stats;
2750 	int i, last_stat_idx = 0, cnt = 0;
2751 
2752 	if (env.out_fmt == RESFMT_TABLE) {
2753 		/* calculate column widths */
2754 		output_headers(RESFMT_TABLE_CALCLEN);
2755 		for (i = 0; i < env.prog_stat_cnt; i++) {
2756 			stats = &env.prog_stats[i];
2757 			if (!should_output_stats(stats))
2758 				continue;
2759 			output_stats(stats, RESFMT_TABLE_CALCLEN, false);
2760 			last_stat_idx = i;
2761 		}
2762 	}
2763 
2764 	/* actually output the table */
2765 	output_headers(env.out_fmt);
2766 	for (i = 0; i < env.prog_stat_cnt; i++) {
2767 		stats = &env.prog_stats[i];
2768 		if (!should_output_stats(stats))
2769 			continue;
2770 		if (env.top_n && cnt >= env.top_n)
2771 			break;
2772 		output_stats(stats, env.out_fmt, i == last_stat_idx);
2773 		cnt++;
2774 	}
2775 }
2776 
2777 static int handle_verif_mode(void)
2778 {
2779 	int i, err;
2780 
2781 	if (env.filename_cnt == 0) {
2782 		fprintf(stderr, "Please provide path to BPF object file!\n\n");
2783 		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
2784 		return -EINVAL;
2785 	}
2786 
2787 	for (i = 0; i < env.filename_cnt; i++) {
2788 		err = process_obj(env.filenames[i]);
2789 		if (err) {
2790 			fprintf(stderr, "Failed to process '%s': %d\n", env.filenames[i], err);
2791 			return err;
2792 		}
2793 	}
2794 
2795 	qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);
2796 
2797 	output_prog_stats();
2798 
2799 	return 0;
2800 }
2801 
2802 static int handle_replay_mode(void)
2803 {
2804 	struct stat_specs specs = {};
2805 	int err;
2806 
2807 	if (env.filename_cnt != 1) {
2808 		fprintf(stderr, "Replay mode expects exactly one input CSV file!\n\n");
2809 		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
2810 		return -EINVAL;
2811 	}
2812 
2813 	err = parse_stats_csv(env.filenames[0], &specs,
2814 			      &env.prog_stats, &env.prog_stat_cnt);
2815 	if (err) {
2816 		fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err);
2817 		return err;
2818 	}
2819 
2820 	qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);
2821 
2822 	output_prog_stats();
2823 
2824 	return 0;
2825 }
2826 
2827 int main(int argc, char **argv)
2828 {
2829 	int err = 0, i;
2830 
2831 	if (argp_parse(&argp, argc, argv, 0, NULL, NULL))
2832 		return 1;
2833 
2834 	if (env.show_version) {
2835 		printf("%s\n", argp_program_version);
2836 		return 0;
2837 	}
2838 
2839 	if (env.verbose && env.quiet) {
2840 		fprintf(stderr, "Verbose and quiet modes are incompatible, please specify just one or neither!\n\n");
2841 		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
2842 		return 1;
2843 	}
2844 	if (env.verbose && env.log_level == 0)
2845 		env.log_level = 1;
2846 
2847 	if (env.output_spec.spec_cnt == 0) {
2848 		if (env.out_fmt == RESFMT_CSV)
2849 			env.output_spec = default_csv_output_spec;
2850 		else
2851 			env.output_spec = default_output_spec;
2852 	}
2853 	if (env.sort_spec.spec_cnt == 0)
2854 		env.sort_spec = default_sort_spec;
2855 
2856 	if (env.comparison_mode && env.replay_mode) {
2857 		fprintf(stderr, "Can't specify replay and comparison mode at the same time!\n\n");
2858 		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
2859 		return 1;
2860 	}
2861 
2862 	if (env.comparison_mode)
2863 		err = handle_comparison_mode();
2864 	else if (env.replay_mode)
2865 		err = handle_replay_mode();
2866 	else
2867 		err = handle_verif_mode();
2868 
2869 	free_verif_stats(env.prog_stats, env.prog_stat_cnt);
2870 	free_verif_stats(env.baseline_stats, env.baseline_stat_cnt);
2871 	free(env.join_stats);
2872 	for (i = 0; i < env.filename_cnt; i++)
2873 		free(env.filenames[i]);
2874 	free(env.filenames);
2875 	for (i = 0; i < env.allow_filter_cnt; i++) {
2876 		free(env.allow_filters[i].any_glob);
2877 		free(env.allow_filters[i].file_glob);
2878 		free(env.allow_filters[i].prog_glob);
2879 	}
2880 	free(env.allow_filters);
2881 	for (i = 0; i < env.deny_filter_cnt; i++) {
2882 		free(env.deny_filters[i].any_glob);
2883 		free(env.deny_filters[i].file_glob);
2884 		free(env.deny_filters[i].prog_glob);
2885 	}
2886 	free(env.deny_filters);
2887 	for (i = 0; i < env.npresets; ++i) {
2888 		free(env.presets[i].name);
2889 		if (env.presets[i].type == ENUMERATOR)
2890 			free(env.presets[i].svalue);
2891 	}
2892 	free(env.presets);
2893 	return -err;
2894 }
2895