xref: /linux/tools/testing/selftests/bpf/test_loader.c (revision 7b26bc6582b13a52a42a4a9765e8f30d58a81198)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3 #include <linux/capability.h>
4 #include <stdlib.h>
5 #include <regex.h>
6 #include <test_progs.h>
7 #include <bpf/btf.h>
8 
9 #include "autoconf_helper.h"
10 #include "disasm_helpers.h"
11 #include "unpriv_helpers.h"
12 #include "cap_helpers.h"
13 #include "jit_disasm_helpers.h"
14 
15 #define str_has_pfx(str, pfx) \
16 	(strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0)
17 
18 #define TEST_LOADER_LOG_BUF_SZ 2097152
19 
20 #define TEST_TAG_EXPECT_FAILURE "comment:test_expect_failure"
21 #define TEST_TAG_EXPECT_SUCCESS "comment:test_expect_success"
22 #define TEST_TAG_EXPECT_MSG_PFX "comment:test_expect_msg="
23 #define TEST_TAG_EXPECT_XLATED_PFX "comment:test_expect_xlated="
24 #define TEST_TAG_EXPECT_FAILURE_UNPRIV "comment:test_expect_failure_unpriv"
25 #define TEST_TAG_EXPECT_SUCCESS_UNPRIV "comment:test_expect_success_unpriv"
26 #define TEST_TAG_EXPECT_MSG_PFX_UNPRIV "comment:test_expect_msg_unpriv="
27 #define TEST_TAG_EXPECT_XLATED_PFX_UNPRIV "comment:test_expect_xlated_unpriv="
28 #define TEST_TAG_LOG_LEVEL_PFX "comment:test_log_level="
29 #define TEST_TAG_PROG_FLAGS_PFX "comment:test_prog_flags="
30 #define TEST_TAG_DESCRIPTION_PFX "comment:test_description="
31 #define TEST_TAG_RETVAL_PFX "comment:test_retval="
32 #define TEST_TAG_RETVAL_PFX_UNPRIV "comment:test_retval_unpriv="
33 #define TEST_TAG_AUXILIARY "comment:test_auxiliary"
34 #define TEST_TAG_AUXILIARY_UNPRIV "comment:test_auxiliary_unpriv"
35 #define TEST_BTF_PATH "comment:test_btf_path="
36 #define TEST_TAG_ARCH "comment:test_arch="
37 #define TEST_TAG_JITED_PFX "comment:test_jited="
38 #define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv="
39 #define TEST_TAG_CAPS_UNPRIV "comment:test_caps_unpriv="
40 
41 /* Warning: duplicated in bpf_misc.h */
42 #define POINTER_VALUE	0xcafe4all
43 #define TEST_DATA_LEN	64
44 
45 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
46 #define EFFICIENT_UNALIGNED_ACCESS 1
47 #else
48 #define EFFICIENT_UNALIGNED_ACCESS 0
49 #endif
50 
51 static int sysctl_unpriv_disabled = -1;
52 
53 enum mode {
54 	PRIV = 1,
55 	UNPRIV = 2
56 };
57 
58 struct expect_msg {
59 	const char *substr; /* substring match */
60 	regex_t regex;
61 	bool is_regex;
62 	bool on_next_line;
63 };
64 
65 struct expected_msgs {
66 	struct expect_msg *patterns;
67 	size_t cnt;
68 };
69 
70 struct test_subspec {
71 	char *name;
72 	bool expect_failure;
73 	struct expected_msgs expect_msgs;
74 	struct expected_msgs expect_xlated;
75 	struct expected_msgs jited;
76 	int retval;
77 	bool execute;
78 	__u64 caps;
79 };
80 
81 struct test_spec {
82 	const char *prog_name;
83 	struct test_subspec priv;
84 	struct test_subspec unpriv;
85 	const char *btf_custom_path;
86 	int log_level;
87 	int prog_flags;
88 	int mode_mask;
89 	int arch_mask;
90 	bool auxiliary;
91 	bool valid;
92 };
93 
94 static int tester_init(struct test_loader *tester)
95 {
96 	if (!tester->log_buf) {
97 		tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ;
98 		tester->log_buf = calloc(tester->log_buf_sz, 1);
99 		if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf"))
100 			return -ENOMEM;
101 	}
102 
103 	return 0;
104 }
105 
106 void test_loader_fini(struct test_loader *tester)
107 {
108 	if (!tester)
109 		return;
110 
111 	free(tester->log_buf);
112 }
113 
114 static void free_msgs(struct expected_msgs *msgs)
115 {
116 	int i;
117 
118 	for (i = 0; i < msgs->cnt; i++)
119 		if (msgs->patterns[i].is_regex)
120 			regfree(&msgs->patterns[i].regex);
121 	free(msgs->patterns);
122 	msgs->patterns = NULL;
123 	msgs->cnt = 0;
124 }
125 
126 static void free_test_spec(struct test_spec *spec)
127 {
128 	/* Deallocate expect_msgs arrays. */
129 	free_msgs(&spec->priv.expect_msgs);
130 	free_msgs(&spec->unpriv.expect_msgs);
131 	free_msgs(&spec->priv.expect_xlated);
132 	free_msgs(&spec->unpriv.expect_xlated);
133 	free_msgs(&spec->priv.jited);
134 	free_msgs(&spec->unpriv.jited);
135 
136 	free(spec->priv.name);
137 	free(spec->unpriv.name);
138 	spec->priv.name = NULL;
139 	spec->unpriv.name = NULL;
140 }
141 
142 /* Compiles regular expression matching pattern.
143  * Pattern has a special syntax:
144  *
145  *   pattern := (<verbatim text> | regex)*
146  *   regex := "{{" <posix extended regular expression> "}}"
147  *
148  * In other words, pattern is a verbatim text with inclusion
149  * of regular expressions enclosed in "{{" "}}" pairs.
150  * For example, pattern "foo{{[0-9]+}}" matches strings like
151  * "foo0", "foo007", etc.
152  */
153 static int compile_regex(const char *pattern, regex_t *regex)
154 {
155 	char err_buf[256], buf[256] = {}, *ptr, *buf_end;
156 	const char *original_pattern = pattern;
157 	bool in_regex = false;
158 	int err;
159 
160 	buf_end = buf + sizeof(buf);
161 	ptr = buf;
162 	while (*pattern && ptr < buf_end - 2) {
163 		if (!in_regex && str_has_pfx(pattern, "{{")) {
164 			in_regex = true;
165 			pattern += 2;
166 			continue;
167 		}
168 		if (in_regex && str_has_pfx(pattern, "}}")) {
169 			in_regex = false;
170 			pattern += 2;
171 			continue;
172 		}
173 		if (in_regex) {
174 			*ptr++ = *pattern++;
175 			continue;
176 		}
177 		/* list of characters that need escaping for extended posix regex */
178 		if (strchr(".[]\\()*+?{}|^$", *pattern)) {
179 			*ptr++ = '\\';
180 			*ptr++ = *pattern++;
181 			continue;
182 		}
183 		*ptr++ = *pattern++;
184 	}
185 	if (*pattern) {
186 		PRINT_FAIL("Regexp too long: '%s'\n", original_pattern);
187 		return -EINVAL;
188 	}
189 	if (in_regex) {
190 		PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern);
191 		return -EINVAL;
192 	}
193 	err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE);
194 	if (err != 0) {
195 		regerror(err, regex, err_buf, sizeof(err_buf));
196 		PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf);
197 		return -EINVAL;
198 	}
199 	return 0;
200 }
201 
202 static int __push_msg(const char *pattern, bool on_next_line, struct expected_msgs *msgs)
203 {
204 	struct expect_msg *msg;
205 	void *tmp;
206 	int err;
207 
208 	tmp = realloc(msgs->patterns,
209 		      (1 + msgs->cnt) * sizeof(struct expect_msg));
210 	if (!tmp) {
211 		ASSERT_FAIL("failed to realloc memory for messages\n");
212 		return -ENOMEM;
213 	}
214 	msgs->patterns = tmp;
215 	msg = &msgs->patterns[msgs->cnt];
216 	msg->on_next_line = on_next_line;
217 	msg->substr = pattern;
218 	msg->is_regex = false;
219 	if (strstr(pattern, "{{")) {
220 		err = compile_regex(pattern, &msg->regex);
221 		if (err)
222 			return err;
223 		msg->is_regex = true;
224 	}
225 	msgs->cnt += 1;
226 	return 0;
227 }
228 
229 static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to)
230 {
231 	struct expect_msg *msg;
232 	int i, err;
233 
234 	for (i = 0; i < from->cnt; i++) {
235 		msg = &from->patterns[i];
236 		err = __push_msg(msg->substr, msg->on_next_line, to);
237 		if (err)
238 			return err;
239 	}
240 	return 0;
241 }
242 
243 static int push_msg(const char *substr, struct expected_msgs *msgs)
244 {
245 	return __push_msg(substr, false, msgs);
246 }
247 
248 static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs)
249 {
250 	int err;
251 
252 	if (strcmp(regex_str, "...") == 0) {
253 		*on_next_line = false;
254 		return 0;
255 	}
256 	err = __push_msg(regex_str, *on_next_line, msgs);
257 	if (err)
258 		return err;
259 	*on_next_line = true;
260 	return 0;
261 }
262 
263 static int parse_int(const char *str, int *val, const char *name)
264 {
265 	char *end;
266 	long tmp;
267 
268 	errno = 0;
269 	if (str_has_pfx(str, "0x"))
270 		tmp = strtol(str + 2, &end, 16);
271 	else
272 		tmp = strtol(str, &end, 10);
273 	if (errno || end[0] != '\0') {
274 		PRINT_FAIL("failed to parse %s from '%s'\n", name, str);
275 		return -EINVAL;
276 	}
277 	*val = tmp;
278 	return 0;
279 }
280 
281 static int parse_caps(const char *str, __u64 *val, const char *name)
282 {
283 	int cap_flag = 0;
284 	char *token = NULL, *saveptr = NULL;
285 
286 	char *str_cpy = strdup(str);
287 	if (str_cpy == NULL) {
288 		PRINT_FAIL("Memory allocation failed\n");
289 		return -EINVAL;
290 	}
291 
292 	token = strtok_r(str_cpy, "|", &saveptr);
293 	while (token != NULL) {
294 		errno = 0;
295 		if (!strncmp("CAP_", token, sizeof("CAP_") - 1)) {
296 			PRINT_FAIL("define %s constant in bpf_misc.h, failed to parse caps\n", token);
297 			return -EINVAL;
298 		}
299 		cap_flag = strtol(token, NULL, 10);
300 		if (!cap_flag || errno) {
301 			PRINT_FAIL("failed to parse caps %s\n", name);
302 			return -EINVAL;
303 		}
304 		*val |= (1ULL << cap_flag);
305 		token = strtok_r(NULL, "|", &saveptr);
306 	}
307 
308 	free(str_cpy);
309 	return 0;
310 }
311 
312 static int parse_retval(const char *str, int *val, const char *name)
313 {
314 	struct {
315 		char *name;
316 		int val;
317 	} named_values[] = {
318 		{ "INT_MIN"      , INT_MIN },
319 		{ "POINTER_VALUE", POINTER_VALUE },
320 		{ "TEST_DATA_LEN", TEST_DATA_LEN },
321 	};
322 	int i;
323 
324 	for (i = 0; i < ARRAY_SIZE(named_values); ++i) {
325 		if (strcmp(str, named_values[i].name) != 0)
326 			continue;
327 		*val = named_values[i].val;
328 		return 0;
329 	}
330 
331 	return parse_int(str, val, name);
332 }
333 
334 static void update_flags(int *flags, int flag, bool clear)
335 {
336 	if (clear)
337 		*flags &= ~flag;
338 	else
339 		*flags |= flag;
340 }
341 
342 /* Matches a string of form '<pfx>[^=]=.*' and returns it's suffix.
343  * Used to parse btf_decl_tag values.
344  * Such values require unique prefix because compiler does not add
345  * same __attribute__((btf_decl_tag(...))) twice.
346  * Test suite uses two-component tags for such cases:
347  *
348  *   <pfx> __COUNTER__ '='
349  *
350  * For example, two consecutive __msg tags '__msg("foo") __msg("foo")'
351  * would be encoded as:
352  *
353  *   [18] DECL_TAG 'comment:test_expect_msg=0=foo' type_id=15 component_idx=-1
354  *   [19] DECL_TAG 'comment:test_expect_msg=1=foo' type_id=15 component_idx=-1
355  *
356  * And the purpose of this function is to extract 'foo' from the above.
357  */
358 static const char *skip_dynamic_pfx(const char *s, const char *pfx)
359 {
360 	const char *msg;
361 
362 	if (strncmp(s, pfx, strlen(pfx)) != 0)
363 		return NULL;
364 	msg = s + strlen(pfx);
365 	msg = strchr(msg, '=');
366 	if (!msg)
367 		return NULL;
368 	return msg + 1;
369 }
370 
371 enum arch {
372 	ARCH_UNKNOWN	= 0x1,
373 	ARCH_X86_64	= 0x2,
374 	ARCH_ARM64	= 0x4,
375 	ARCH_RISCV64	= 0x8,
376 };
377 
378 static int get_current_arch(void)
379 {
380 #if defined(__x86_64__)
381 	return ARCH_X86_64;
382 #elif defined(__aarch64__)
383 	return ARCH_ARM64;
384 #elif defined(__riscv) && __riscv_xlen == 64
385 	return ARCH_RISCV64;
386 #endif
387 	return ARCH_UNKNOWN;
388 }
389 
390 /* Uses btf_decl_tag attributes to describe the expected test
391  * behavior, see bpf_misc.h for detailed description of each attribute
392  * and attribute combinations.
393  */
394 static int parse_test_spec(struct test_loader *tester,
395 			   struct bpf_object *obj,
396 			   struct bpf_program *prog,
397 			   struct test_spec *spec)
398 {
399 	const char *description = NULL;
400 	bool has_unpriv_result = false;
401 	bool has_unpriv_retval = false;
402 	bool unpriv_xlated_on_next_line = true;
403 	bool xlated_on_next_line = true;
404 	bool unpriv_jit_on_next_line;
405 	bool jit_on_next_line;
406 	bool collect_jit = false;
407 	int func_id, i, err = 0;
408 	u32 arch_mask = 0;
409 	struct btf *btf;
410 	enum arch arch;
411 
412 	memset(spec, 0, sizeof(*spec));
413 
414 	spec->prog_name = bpf_program__name(prog);
415 	spec->prog_flags = testing_prog_flags();
416 
417 	btf = bpf_object__btf(obj);
418 	if (!btf) {
419 		ASSERT_FAIL("BPF object has no BTF");
420 		return -EINVAL;
421 	}
422 
423 	func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC);
424 	if (func_id < 0) {
425 		ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name);
426 		return -EINVAL;
427 	}
428 
429 	for (i = 1; i < btf__type_cnt(btf); i++) {
430 		const char *s, *val, *msg;
431 		const struct btf_type *t;
432 		bool clear;
433 		int flags;
434 
435 		t = btf__type_by_id(btf, i);
436 		if (!btf_is_decl_tag(t))
437 			continue;
438 
439 		if (t->type != func_id || btf_decl_tag(t)->component_idx != -1)
440 			continue;
441 
442 		s = btf__str_by_offset(btf, t->name_off);
443 		if (str_has_pfx(s, TEST_TAG_DESCRIPTION_PFX)) {
444 			description = s + sizeof(TEST_TAG_DESCRIPTION_PFX) - 1;
445 		} else if (strcmp(s, TEST_TAG_EXPECT_FAILURE) == 0) {
446 			spec->priv.expect_failure = true;
447 			spec->mode_mask |= PRIV;
448 		} else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS) == 0) {
449 			spec->priv.expect_failure = false;
450 			spec->mode_mask |= PRIV;
451 		} else if (strcmp(s, TEST_TAG_EXPECT_FAILURE_UNPRIV) == 0) {
452 			spec->unpriv.expect_failure = true;
453 			spec->mode_mask |= UNPRIV;
454 			has_unpriv_result = true;
455 		} else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS_UNPRIV) == 0) {
456 			spec->unpriv.expect_failure = false;
457 			spec->mode_mask |= UNPRIV;
458 			has_unpriv_result = true;
459 		} else if (strcmp(s, TEST_TAG_AUXILIARY) == 0) {
460 			spec->auxiliary = true;
461 			spec->mode_mask |= PRIV;
462 		} else if (strcmp(s, TEST_TAG_AUXILIARY_UNPRIV) == 0) {
463 			spec->auxiliary = true;
464 			spec->mode_mask |= UNPRIV;
465 		} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX))) {
466 			err = push_msg(msg, &spec->priv.expect_msgs);
467 			if (err)
468 				goto cleanup;
469 			spec->mode_mask |= PRIV;
470 		} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX_UNPRIV))) {
471 			err = push_msg(msg, &spec->unpriv.expect_msgs);
472 			if (err)
473 				goto cleanup;
474 			spec->mode_mask |= UNPRIV;
475 		} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX))) {
476 			if (arch_mask == 0) {
477 				PRINT_FAIL("__jited used before __arch_*");
478 				goto cleanup;
479 			}
480 			if (collect_jit) {
481 				err = push_disasm_msg(msg, &jit_on_next_line,
482 						      &spec->priv.jited);
483 				if (err)
484 					goto cleanup;
485 				spec->mode_mask |= PRIV;
486 			}
487 		} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX_UNPRIV))) {
488 			if (arch_mask == 0) {
489 				PRINT_FAIL("__unpriv_jited used before __arch_*");
490 				goto cleanup;
491 			}
492 			if (collect_jit) {
493 				err = push_disasm_msg(msg, &unpriv_jit_on_next_line,
494 						      &spec->unpriv.jited);
495 				if (err)
496 					goto cleanup;
497 				spec->mode_mask |= UNPRIV;
498 			}
499 		} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX))) {
500 			err = push_disasm_msg(msg, &xlated_on_next_line,
501 					      &spec->priv.expect_xlated);
502 			if (err)
503 				goto cleanup;
504 			spec->mode_mask |= PRIV;
505 		} else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX_UNPRIV))) {
506 			err = push_disasm_msg(msg, &unpriv_xlated_on_next_line,
507 					      &spec->unpriv.expect_xlated);
508 			if (err)
509 				goto cleanup;
510 			spec->mode_mask |= UNPRIV;
511 		} else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX)) {
512 			val = s + sizeof(TEST_TAG_RETVAL_PFX) - 1;
513 			err = parse_retval(val, &spec->priv.retval, "__retval");
514 			if (err)
515 				goto cleanup;
516 			spec->priv.execute = true;
517 			spec->mode_mask |= PRIV;
518 		} else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX_UNPRIV)) {
519 			val = s + sizeof(TEST_TAG_RETVAL_PFX_UNPRIV) - 1;
520 			err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv");
521 			if (err)
522 				goto cleanup;
523 			spec->mode_mask |= UNPRIV;
524 			spec->unpriv.execute = true;
525 			has_unpriv_retval = true;
526 		} else if (str_has_pfx(s, TEST_TAG_LOG_LEVEL_PFX)) {
527 			val = s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1;
528 			err = parse_int(val, &spec->log_level, "test log level");
529 			if (err)
530 				goto cleanup;
531 		} else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) {
532 			val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1;
533 
534 			clear = val[0] == '!';
535 			if (clear)
536 				val++;
537 
538 			if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) {
539 				update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear);
540 			} else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) {
541 				update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear);
542 			} else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) {
543 				update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear);
544 			} else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) {
545 				update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear);
546 			} else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) {
547 				update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear);
548 			} else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) {
549 				update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear);
550 			} else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) {
551 				update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear);
552 			} else /* assume numeric value */ {
553 				err = parse_int(val, &flags, "test prog flags");
554 				if (err)
555 					goto cleanup;
556 				update_flags(&spec->prog_flags, flags, clear);
557 			}
558 		} else if (str_has_pfx(s, TEST_TAG_ARCH)) {
559 			val = s + sizeof(TEST_TAG_ARCH) - 1;
560 			if (strcmp(val, "X86_64") == 0) {
561 				arch = ARCH_X86_64;
562 			} else if (strcmp(val, "ARM64") == 0) {
563 				arch = ARCH_ARM64;
564 			} else if (strcmp(val, "RISCV64") == 0) {
565 				arch = ARCH_RISCV64;
566 			} else {
567 				PRINT_FAIL("bad arch spec: '%s'", val);
568 				err = -EINVAL;
569 				goto cleanup;
570 			}
571 			arch_mask |= arch;
572 			collect_jit = get_current_arch() == arch;
573 			unpriv_jit_on_next_line = true;
574 			jit_on_next_line = true;
575 		} else if (str_has_pfx(s, TEST_BTF_PATH)) {
576 			spec->btf_custom_path = s + sizeof(TEST_BTF_PATH) - 1;
577 		} else if (str_has_pfx(s, TEST_TAG_CAPS_UNPRIV)) {
578 			val = s + sizeof(TEST_TAG_CAPS_UNPRIV) - 1;
579 			err = parse_caps(val, &spec->unpriv.caps, "test caps");
580 			if (err)
581 				goto cleanup;
582 			spec->mode_mask |= UNPRIV;
583 		}
584 	}
585 
586 	spec->arch_mask = arch_mask ?: -1;
587 
588 	if (spec->mode_mask == 0)
589 		spec->mode_mask = PRIV;
590 
591 	if (!description)
592 		description = spec->prog_name;
593 
594 	if (spec->mode_mask & PRIV) {
595 		spec->priv.name = strdup(description);
596 		if (!spec->priv.name) {
597 			PRINT_FAIL("failed to allocate memory for priv.name\n");
598 			err = -ENOMEM;
599 			goto cleanup;
600 		}
601 	}
602 
603 	if (spec->mode_mask & UNPRIV) {
604 		int descr_len = strlen(description);
605 		const char *suffix = " @unpriv";
606 		char *name;
607 
608 		name = malloc(descr_len + strlen(suffix) + 1);
609 		if (!name) {
610 			PRINT_FAIL("failed to allocate memory for unpriv.name\n");
611 			err = -ENOMEM;
612 			goto cleanup;
613 		}
614 
615 		strcpy(name, description);
616 		strcpy(&name[descr_len], suffix);
617 		spec->unpriv.name = name;
618 	}
619 
620 	if (spec->mode_mask & (PRIV | UNPRIV)) {
621 		if (!has_unpriv_result)
622 			spec->unpriv.expect_failure = spec->priv.expect_failure;
623 
624 		if (!has_unpriv_retval) {
625 			spec->unpriv.retval = spec->priv.retval;
626 			spec->unpriv.execute = spec->priv.execute;
627 		}
628 
629 		if (spec->unpriv.expect_msgs.cnt == 0)
630 			clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs);
631 		if (spec->unpriv.expect_xlated.cnt == 0)
632 			clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated);
633 		if (spec->unpriv.jited.cnt == 0)
634 			clone_msgs(&spec->priv.jited, &spec->unpriv.jited);
635 	}
636 
637 	spec->valid = true;
638 
639 	return 0;
640 
641 cleanup:
642 	free_test_spec(spec);
643 	return err;
644 }
645 
646 static void prepare_case(struct test_loader *tester,
647 			 struct test_spec *spec,
648 			 struct bpf_object *obj,
649 			 struct bpf_program *prog)
650 {
651 	int min_log_level = 0, prog_flags;
652 
653 	if (env.verbosity > VERBOSE_NONE)
654 		min_log_level = 1;
655 	if (env.verbosity > VERBOSE_VERY)
656 		min_log_level = 2;
657 
658 	bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz);
659 
660 	/* Make sure we set at least minimal log level, unless test requires
661 	 * even higher level already. Make sure to preserve independent log
662 	 * level 4 (verifier stats), though.
663 	 */
664 	if ((spec->log_level & 3) < min_log_level)
665 		bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level);
666 	else
667 		bpf_program__set_log_level(prog, spec->log_level);
668 
669 	prog_flags = bpf_program__flags(prog);
670 	bpf_program__set_flags(prog, prog_flags | spec->prog_flags);
671 
672 	tester->log_buf[0] = '\0';
673 }
674 
675 static void emit_verifier_log(const char *log_buf, bool force)
676 {
677 	if (!force && env.verbosity == VERBOSE_NONE)
678 		return;
679 	fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf);
680 }
681 
682 static void emit_xlated(const char *xlated, bool force)
683 {
684 	if (!force && env.verbosity == VERBOSE_NONE)
685 		return;
686 	fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated);
687 }
688 
689 static void emit_jited(const char *jited, bool force)
690 {
691 	if (!force && env.verbosity == VERBOSE_NONE)
692 		return;
693 	fprintf(stdout, "JITED:\n=============\n%s=============\n", jited);
694 }
695 
696 static void validate_msgs(char *log_buf, struct expected_msgs *msgs,
697 			  void (*emit_fn)(const char *buf, bool force))
698 {
699 	const char *log = log_buf, *prev_match;
700 	regmatch_t reg_match[1];
701 	int prev_match_line;
702 	int match_line;
703 	int i, j, err;
704 
705 	prev_match_line = -1;
706 	match_line = 0;
707 	prev_match = log;
708 	for (i = 0; i < msgs->cnt; i++) {
709 		struct expect_msg *msg = &msgs->patterns[i];
710 		const char *match = NULL, *pat_status;
711 		bool wrong_line = false;
712 
713 		if (!msg->is_regex) {
714 			match = strstr(log, msg->substr);
715 			if (match)
716 				log = match + strlen(msg->substr);
717 		} else {
718 			err = regexec(&msg->regex, log, 1, reg_match, 0);
719 			if (err == 0) {
720 				match = log + reg_match[0].rm_so;
721 				log += reg_match[0].rm_eo;
722 			}
723 		}
724 
725 		if (match) {
726 			for (; prev_match < match; ++prev_match)
727 				if (*prev_match == '\n')
728 					++match_line;
729 			wrong_line = msg->on_next_line && prev_match_line >= 0 &&
730 				     prev_match_line + 1 != match_line;
731 		}
732 
733 		if (!match || wrong_line) {
734 			PRINT_FAIL("expect_msg\n");
735 			if (env.verbosity == VERBOSE_NONE)
736 				emit_fn(log_buf, true /*force*/);
737 			for (j = 0; j <= i; j++) {
738 				msg = &msgs->patterns[j];
739 				if (j < i)
740 					pat_status = "MATCHED   ";
741 				else if (wrong_line)
742 					pat_status = "WRONG LINE";
743 				else
744 					pat_status = "EXPECTED  ";
745 				msg = &msgs->patterns[j];
746 				fprintf(stderr, "%s %s: '%s'\n",
747 					pat_status,
748 					msg->is_regex ? " REGEX" : "SUBSTR",
749 					msg->substr);
750 			}
751 			if (wrong_line) {
752 				fprintf(stderr,
753 					"expecting match at line %d, actual match is at line %d\n",
754 					prev_match_line + 1, match_line);
755 			}
756 			break;
757 		}
758 
759 		prev_match_line = match_line;
760 	}
761 }
762 
763 struct cap_state {
764 	__u64 old_caps;
765 	bool initialized;
766 };
767 
768 static int drop_capabilities(struct cap_state *caps)
769 {
770 	const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN |
771 				    1ULL << CAP_PERFMON   | 1ULL << CAP_BPF);
772 	int err;
773 
774 	err = cap_disable_effective(caps_to_drop, &caps->old_caps);
775 	if (err) {
776 		PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(err));
777 		return err;
778 	}
779 
780 	caps->initialized = true;
781 	return 0;
782 }
783 
784 static int restore_capabilities(struct cap_state *caps)
785 {
786 	int err;
787 
788 	if (!caps->initialized)
789 		return 0;
790 
791 	err = cap_enable_effective(caps->old_caps, NULL);
792 	if (err)
793 		PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(err));
794 	caps->initialized = false;
795 	return err;
796 }
797 
798 static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec)
799 {
800 	if (sysctl_unpriv_disabled < 0)
801 		sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0;
802 	if (sysctl_unpriv_disabled)
803 		return false;
804 	if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS)
805 		return false;
806 	return true;
807 }
808 
809 static bool is_unpriv_capable_map(struct bpf_map *map)
810 {
811 	enum bpf_map_type type;
812 	__u32 flags;
813 
814 	type = bpf_map__type(map);
815 
816 	switch (type) {
817 	case BPF_MAP_TYPE_HASH:
818 	case BPF_MAP_TYPE_PERCPU_HASH:
819 	case BPF_MAP_TYPE_HASH_OF_MAPS:
820 		flags = bpf_map__map_flags(map);
821 		return !(flags & BPF_F_ZERO_SEED);
822 	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
823 	case BPF_MAP_TYPE_ARRAY:
824 	case BPF_MAP_TYPE_RINGBUF:
825 	case BPF_MAP_TYPE_PROG_ARRAY:
826 	case BPF_MAP_TYPE_CGROUP_ARRAY:
827 	case BPF_MAP_TYPE_PERCPU_ARRAY:
828 	case BPF_MAP_TYPE_USER_RINGBUF:
829 	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
830 	case BPF_MAP_TYPE_CGROUP_STORAGE:
831 	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
832 		return true;
833 	default:
834 		return false;
835 	}
836 }
837 
838 static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts)
839 {
840 	__u8 tmp_out[TEST_DATA_LEN << 2] = {};
841 	__u8 tmp_in[TEST_DATA_LEN] = {};
842 	int err, saved_errno;
843 	LIBBPF_OPTS(bpf_test_run_opts, topts,
844 		.data_in = tmp_in,
845 		.data_size_in = sizeof(tmp_in),
846 		.data_out = tmp_out,
847 		.data_size_out = sizeof(tmp_out),
848 		.repeat = 1,
849 	);
850 
851 	if (empty_opts) {
852 		memset(&topts, 0, sizeof(struct bpf_test_run_opts));
853 		topts.sz = sizeof(struct bpf_test_run_opts);
854 	}
855 	err = bpf_prog_test_run_opts(fd_prog, &topts);
856 	saved_errno = errno;
857 
858 	if (err) {
859 		PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ",
860 			   saved_errno, strerror(saved_errno));
861 		return err;
862 	}
863 
864 	ASSERT_OK(0, "bpf_prog_test_run");
865 	*retval = topts.retval;
866 
867 	return 0;
868 }
869 
870 static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec)
871 {
872 	if (!subspec->execute)
873 		return false;
874 
875 	if (subspec->expect_failure)
876 		return false;
877 
878 	if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) {
879 		if (env.verbosity != VERBOSE_NONE)
880 			printf("alignment prevents execution\n");
881 		return false;
882 	}
883 
884 	return true;
885 }
886 
887 /* Get a disassembly of BPF program after verifier applies all rewrites */
888 static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz)
889 {
890 	struct bpf_insn *insn_start = NULL, *insn, *insn_end;
891 	__u32 insns_cnt = 0, i;
892 	char buf[64];
893 	FILE *out = NULL;
894 	int err;
895 
896 	err = get_xlated_program(prog_fd, &insn_start, &insns_cnt);
897 	if (!ASSERT_OK(err, "get_xlated_program"))
898 		goto out;
899 	out = fmemopen(text, text_sz, "w");
900 	if (!ASSERT_OK_PTR(out, "open_memstream"))
901 		goto out;
902 	insn_end = insn_start + insns_cnt;
903 	insn = insn_start;
904 	while (insn < insn_end) {
905 		i = insn - insn_start;
906 		insn = disasm_insn(insn, buf, sizeof(buf));
907 		fprintf(out, "%d: %s\n", i, buf);
908 	}
909 	fflush(out);
910 
911 out:
912 	free(insn_start);
913 	if (out)
914 		fclose(out);
915 	return err;
916 }
917 
918 /* this function is forced noinline and has short generic name to look better
919  * in test_progs output (in case of a failure)
920  */
921 static noinline
922 void run_subtest(struct test_loader *tester,
923 		 struct bpf_object_open_opts *open_opts,
924 		 const void *obj_bytes,
925 		 size_t obj_byte_cnt,
926 		 struct test_spec *specs,
927 		 struct test_spec *spec,
928 		 bool unpriv)
929 {
930 	struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv;
931 	struct bpf_program *tprog = NULL, *tprog_iter;
932 	struct bpf_link *link, *links[32] = {};
933 	struct test_spec *spec_iter;
934 	struct cap_state caps = {};
935 	struct bpf_object *tobj;
936 	struct bpf_map *map;
937 	int retval, err, i;
938 	int links_cnt = 0;
939 	bool should_load;
940 
941 	if (!test__start_subtest(subspec->name))
942 		return;
943 
944 	if ((get_current_arch() & spec->arch_mask) == 0) {
945 		test__skip();
946 		return;
947 	}
948 
949 	if (unpriv) {
950 		if (!can_execute_unpriv(tester, spec)) {
951 			test__skip();
952 			test__end_subtest();
953 			return;
954 		}
955 		if (drop_capabilities(&caps)) {
956 			test__end_subtest();
957 			return;
958 		}
959 		if (subspec->caps) {
960 			err = cap_enable_effective(subspec->caps, NULL);
961 			if (err) {
962 				PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(err));
963 				goto subtest_cleanup;
964 			}
965 		}
966 	}
967 
968 	/* Implicitly reset to NULL if next test case doesn't specify */
969 	open_opts->btf_custom_path = spec->btf_custom_path;
970 
971 	tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts);
972 	if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */
973 		goto subtest_cleanup;
974 
975 	i = 0;
976 	bpf_object__for_each_program(tprog_iter, tobj) {
977 		spec_iter = &specs[i++];
978 		should_load = false;
979 
980 		if (spec_iter->valid) {
981 			if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) {
982 				tprog = tprog_iter;
983 				should_load = true;
984 			}
985 
986 			if (spec_iter->auxiliary &&
987 			    spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV))
988 				should_load = true;
989 		}
990 
991 		bpf_program__set_autoload(tprog_iter, should_load);
992 	}
993 
994 	prepare_case(tester, spec, tobj, tprog);
995 
996 	/* By default bpf_object__load() automatically creates all
997 	 * maps declared in the skeleton. Some map types are only
998 	 * allowed in priv mode. Disable autoload for such maps in
999 	 * unpriv mode.
1000 	 */
1001 	bpf_object__for_each_map(map, tobj)
1002 		bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map));
1003 
1004 	err = bpf_object__load(tobj);
1005 	if (subspec->expect_failure) {
1006 		if (!ASSERT_ERR(err, "unexpected_load_success")) {
1007 			emit_verifier_log(tester->log_buf, false /*force*/);
1008 			goto tobj_cleanup;
1009 		}
1010 	} else {
1011 		if (!ASSERT_OK(err, "unexpected_load_failure")) {
1012 			emit_verifier_log(tester->log_buf, true /*force*/);
1013 			goto tobj_cleanup;
1014 		}
1015 	}
1016 	emit_verifier_log(tester->log_buf, false /*force*/);
1017 	validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log);
1018 
1019 	if (subspec->expect_xlated.cnt) {
1020 		err = get_xlated_program_text(bpf_program__fd(tprog),
1021 					      tester->log_buf, tester->log_buf_sz);
1022 		if (err)
1023 			goto tobj_cleanup;
1024 		emit_xlated(tester->log_buf, false /*force*/);
1025 		validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated);
1026 	}
1027 
1028 	if (subspec->jited.cnt) {
1029 		err = get_jited_program_text(bpf_program__fd(tprog),
1030 					     tester->log_buf, tester->log_buf_sz);
1031 		if (err == -EOPNOTSUPP) {
1032 			printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__);
1033 			printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__);
1034 			test__skip();
1035 			goto tobj_cleanup;
1036 		}
1037 		if (!ASSERT_EQ(err, 0, "get_jited_program_text"))
1038 			goto tobj_cleanup;
1039 		emit_jited(tester->log_buf, false /*force*/);
1040 		validate_msgs(tester->log_buf, &subspec->jited, emit_jited);
1041 	}
1042 
1043 	if (should_do_test_run(spec, subspec)) {
1044 		/* For some reason test_verifier executes programs
1045 		 * with all capabilities restored. Do the same here.
1046 		 */
1047 		if (restore_capabilities(&caps))
1048 			goto tobj_cleanup;
1049 
1050 		/* Do bpf_map__attach_struct_ops() for each struct_ops map.
1051 		 * This should trigger bpf_struct_ops->reg callback on kernel side.
1052 		 */
1053 		bpf_object__for_each_map(map, tobj) {
1054 			if (!bpf_map__autocreate(map) ||
1055 			    bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS)
1056 				continue;
1057 			if (links_cnt >= ARRAY_SIZE(links)) {
1058 				PRINT_FAIL("too many struct_ops maps");
1059 				goto tobj_cleanup;
1060 			}
1061 			link = bpf_map__attach_struct_ops(map);
1062 			if (!link) {
1063 				PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n",
1064 					   bpf_map__name(map), err);
1065 				goto tobj_cleanup;
1066 			}
1067 			links[links_cnt++] = link;
1068 		}
1069 
1070 		if (tester->pre_execution_cb) {
1071 			err = tester->pre_execution_cb(tobj);
1072 			if (err) {
1073 				PRINT_FAIL("pre_execution_cb failed: %d\n", err);
1074 				goto tobj_cleanup;
1075 			}
1076 		}
1077 
1078 		do_prog_test_run(bpf_program__fd(tprog), &retval,
1079 				 bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false);
1080 		if (retval != subspec->retval && subspec->retval != POINTER_VALUE) {
1081 			PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval);
1082 			goto tobj_cleanup;
1083 		}
1084 		/* redo bpf_map__attach_struct_ops for each test */
1085 		while (links_cnt > 0)
1086 			bpf_link__destroy(links[--links_cnt]);
1087 	}
1088 
1089 tobj_cleanup:
1090 	while (links_cnt > 0)
1091 		bpf_link__destroy(links[--links_cnt]);
1092 	bpf_object__close(tobj);
1093 subtest_cleanup:
1094 	test__end_subtest();
1095 	restore_capabilities(&caps);
1096 }
1097 
1098 static void process_subtest(struct test_loader *tester,
1099 			    const char *skel_name,
1100 			    skel_elf_bytes_fn elf_bytes_factory)
1101 {
1102 	LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name);
1103 	struct test_spec *specs = NULL;
1104 	struct bpf_object *obj = NULL;
1105 	struct bpf_program *prog;
1106 	const void *obj_bytes;
1107 	int err, i, nr_progs;
1108 	size_t obj_byte_cnt;
1109 
1110 	if (tester_init(tester) < 0)
1111 		return; /* failed to initialize tester */
1112 
1113 	obj_bytes = elf_bytes_factory(&obj_byte_cnt);
1114 	obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts);
1115 	if (!ASSERT_OK_PTR(obj, "obj_open_mem"))
1116 		return;
1117 
1118 	nr_progs = 0;
1119 	bpf_object__for_each_program(prog, obj)
1120 		++nr_progs;
1121 
1122 	specs = calloc(nr_progs, sizeof(struct test_spec));
1123 	if (!ASSERT_OK_PTR(specs, "specs_alloc"))
1124 		return;
1125 
1126 	i = 0;
1127 	bpf_object__for_each_program(prog, obj) {
1128 		/* ignore tests for which  we can't derive test specification */
1129 		err = parse_test_spec(tester, obj, prog, &specs[i++]);
1130 		if (err)
1131 			PRINT_FAIL("Can't parse test spec for program '%s'\n",
1132 				   bpf_program__name(prog));
1133 	}
1134 
1135 	i = 0;
1136 	bpf_object__for_each_program(prog, obj) {
1137 		struct test_spec *spec = &specs[i++];
1138 
1139 		if (!spec->valid || spec->auxiliary)
1140 			continue;
1141 
1142 		if (spec->mode_mask & PRIV)
1143 			run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
1144 				    specs, spec, false);
1145 		if (spec->mode_mask & UNPRIV)
1146 			run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
1147 				    specs, spec, true);
1148 
1149 	}
1150 
1151 	for (i = 0; i < nr_progs; ++i)
1152 		free_test_spec(&specs[i]);
1153 	free(specs);
1154 	bpf_object__close(obj);
1155 }
1156 
1157 void test_loader__run_subtests(struct test_loader *tester,
1158 			       const char *skel_name,
1159 			       skel_elf_bytes_fn elf_bytes_factory)
1160 {
1161 	/* see comment in run_subtest() for why we do this function nesting */
1162 	process_subtest(tester, skel_name, elf_bytes_factory);
1163 }
1164