1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3 #include <linux/capability.h>
4 #include <linux/err.h>
5 #include <stdlib.h>
6 #include <test_progs.h>
7 #include <bpf/btf.h>
8
9 #include "autoconf_helper.h"
10 #include "disasm_helpers.h"
11 #include "unpriv_helpers.h"
12 #include "cap_helpers.h"
13 #include "jit_disasm_helpers.h"
14
str_has_pfx(const char * str,const char * pfx)15 static inline const char *str_has_pfx(const char *str, const char *pfx)
16 {
17 size_t len = strlen(pfx);
18
19 return strncmp(str, pfx, len) == 0 ? str + len : NULL;
20 }
21
22 #define TEST_LOADER_LOG_BUF_SZ 2097152
23
24
25 /* Warning: duplicated in bpf_misc.h */
26 #define POINTER_VALUE 0xbadcafe
27 #define TEST_DATA_LEN 64
28
29 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
30 #define EFFICIENT_UNALIGNED_ACCESS 1
31 #else
32 #define EFFICIENT_UNALIGNED_ACCESS 0
33 #endif
34
35 static int sysctl_unpriv_disabled = -1;
36
37 enum mode {
38 PRIV = 1,
39 UNPRIV = 2
40 };
41
42 enum load_mode {
43 JITED = 1 << 0,
44 NO_JITED = 1 << 1,
45 };
46
47 struct test_subspec {
48 char *name;
49 char *description;
50 bool expect_failure;
51 struct expected_msgs expect_msgs;
52 struct expected_msgs expect_xlated;
53 struct expected_msgs jited;
54 struct expected_msgs stderr;
55 struct expected_msgs stdout;
56 int retval;
57 bool execute;
58 __u64 caps;
59 };
60
61 struct test_spec {
62 const char *prog_name;
63 struct test_subspec priv;
64 struct test_subspec unpriv;
65 const char *btf_custom_path;
66 const char *btf_custom_func_path;
67 int log_level;
68 int prog_flags;
69 int mode_mask;
70 int arch_mask;
71 int load_mask;
72 int linear_sz;
73 bool auxiliary;
74 bool valid;
75 };
76
tester_init(struct test_loader * tester)77 static int tester_init(struct test_loader *tester)
78 {
79 if (!tester->log_buf) {
80 tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ;
81 tester->log_buf = calloc(tester->log_buf_sz, 1);
82 if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf"))
83 return -ENOMEM;
84 }
85
86 return 0;
87 }
88
test_loader_fini(struct test_loader * tester)89 void test_loader_fini(struct test_loader *tester)
90 {
91 if (!tester)
92 return;
93
94 free(tester->log_buf);
95 }
96
free_msgs(struct expected_msgs * msgs)97 void free_msgs(struct expected_msgs *msgs)
98 {
99 int i;
100
101 for (i = 0; i < msgs->cnt; i++)
102 if (msgs->patterns[i].is_regex)
103 regfree(&msgs->patterns[i].regex);
104 free(msgs->patterns);
105 msgs->patterns = NULL;
106 msgs->cnt = 0;
107 }
108
free_test_spec(struct test_spec * spec)109 static void free_test_spec(struct test_spec *spec)
110 {
111 /* Deallocate expect_msgs arrays. */
112 free_msgs(&spec->priv.expect_msgs);
113 free_msgs(&spec->unpriv.expect_msgs);
114 free_msgs(&spec->priv.expect_xlated);
115 free_msgs(&spec->unpriv.expect_xlated);
116 free_msgs(&spec->priv.jited);
117 free_msgs(&spec->unpriv.jited);
118 free_msgs(&spec->unpriv.stderr);
119 free_msgs(&spec->priv.stderr);
120 free_msgs(&spec->unpriv.stdout);
121 free_msgs(&spec->priv.stdout);
122
123 free(spec->priv.name);
124 free(spec->priv.description);
125 free(spec->unpriv.name);
126 free(spec->unpriv.description);
127 spec->priv.name = NULL;
128 spec->priv.description = NULL;
129 spec->unpriv.name = NULL;
130 spec->unpriv.description = NULL;
131 }
132
133 /* Compiles regular expression matching pattern.
134 * Pattern has a special syntax:
135 *
136 * pattern := (<verbatim text> | regex)*
137 * regex := "{{" <posix extended regular expression> "}}"
138 *
139 * In other words, pattern is a verbatim text with inclusion
140 * of regular expressions enclosed in "{{" "}}" pairs.
141 * For example, pattern "foo{{[0-9]+}}" matches strings like
142 * "foo0", "foo007", etc.
143 */
compile_regex(const char * pattern,regex_t * regex)144 static int compile_regex(const char *pattern, regex_t *regex)
145 {
146 char err_buf[256], buf[256] = {}, *ptr, *buf_end;
147 const char *original_pattern = pattern, *next;
148 bool in_regex = false;
149 int err;
150
151 buf_end = buf + sizeof(buf);
152 ptr = buf;
153 while (*pattern && ptr < buf_end - 2) {
154 if (!in_regex && (next = str_has_pfx(pattern, "{{"))) {
155 in_regex = true;
156 pattern = next;
157 continue;
158 }
159 if (in_regex && (next = str_has_pfx(pattern, "}}"))) {
160 in_regex = false;
161 pattern = next;
162 continue;
163 }
164 if (in_regex) {
165 *ptr++ = *pattern++;
166 continue;
167 }
168 /* list of characters that need escaping for extended posix regex */
169 if (strchr(".[]\\()*+?{}|^$", *pattern)) {
170 *ptr++ = '\\';
171 *ptr++ = *pattern++;
172 continue;
173 }
174 *ptr++ = *pattern++;
175 }
176 if (*pattern) {
177 PRINT_FAIL("Regexp too long: '%s'\n", original_pattern);
178 return -EINVAL;
179 }
180 if (in_regex) {
181 PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern);
182 return -EINVAL;
183 }
184 err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE);
185 if (err != 0) {
186 regerror(err, regex, err_buf, sizeof(err_buf));
187 PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf);
188 return -EINVAL;
189 }
190 return 0;
191 }
192
__push_msg(const char * pattern,bool on_next_line,bool negative,struct expected_msgs * msgs)193 static int __push_msg(const char *pattern, bool on_next_line, bool negative,
194 struct expected_msgs *msgs)
195 {
196 struct expect_msg *msg;
197 void *tmp;
198 int err;
199
200 tmp = realloc(msgs->patterns,
201 (1 + msgs->cnt) * sizeof(struct expect_msg));
202 if (!tmp) {
203 ASSERT_FAIL("failed to realloc memory for messages\n");
204 return -ENOMEM;
205 }
206 msgs->patterns = tmp;
207 msg = &msgs->patterns[msgs->cnt];
208 msg->on_next_line = on_next_line;
209 msg->substr = pattern;
210 msg->negative = negative;
211 msg->is_regex = false;
212 if (strstr(pattern, "{{")) {
213 err = compile_regex(pattern, &msg->regex);
214 if (err)
215 return err;
216 msg->is_regex = true;
217 }
218 msgs->cnt += 1;
219 return 0;
220 }
221
clone_msgs(struct expected_msgs * from,struct expected_msgs * to)222 static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to)
223 {
224 struct expect_msg *msg;
225 int i, err;
226
227 for (i = 0; i < from->cnt; i++) {
228 msg = &from->patterns[i];
229 err = __push_msg(msg->substr, msg->on_next_line, msg->negative, to);
230 if (err)
231 return err;
232 }
233 return 0;
234 }
235
push_msg(const char * substr,bool negative,struct expected_msgs * msgs)236 static int push_msg(const char *substr, bool negative, struct expected_msgs *msgs)
237 {
238 return __push_msg(substr, false, negative, msgs);
239 }
240
push_disasm_msg(const char * regex_str,bool * on_next_line,struct expected_msgs * msgs)241 static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs)
242 {
243 int err;
244
245 if (strcmp(regex_str, "...") == 0) {
246 *on_next_line = false;
247 return 0;
248 }
249 err = __push_msg(regex_str, *on_next_line, false, msgs);
250 if (err)
251 return err;
252 *on_next_line = true;
253 return 0;
254 }
255
parse_int(const char * str,int * val,const char * name)256 static int parse_int(const char *str, int *val, const char *name)
257 {
258 char *end;
259 long tmp;
260
261 errno = 0;
262 if (str_has_pfx(str, "0x"))
263 tmp = strtol(str + 2, &end, 16);
264 else
265 tmp = strtol(str, &end, 10);
266 if (errno || end[0] != '\0') {
267 PRINT_FAIL("failed to parse %s from '%s'\n", name, str);
268 return -EINVAL;
269 }
270 *val = tmp;
271 return 0;
272 }
273
parse_caps(const char * str,__u64 * val,const char * name)274 static int parse_caps(const char *str, __u64 *val, const char *name)
275 {
276 int cap_flag = 0;
277 char *token = NULL, *saveptr = NULL;
278
279 char *str_cpy = strdup(str);
280 if (str_cpy == NULL) {
281 PRINT_FAIL("Memory allocation failed\n");
282 return -EINVAL;
283 }
284
285 token = strtok_r(str_cpy, "|", &saveptr);
286 while (token != NULL) {
287 errno = 0;
288 if (!strncmp("CAP_", token, sizeof("CAP_") - 1)) {
289 PRINT_FAIL("define %s constant in bpf_misc.h, failed to parse caps\n", token);
290 return -EINVAL;
291 }
292 cap_flag = strtol(token, NULL, 10);
293 if (!cap_flag || errno) {
294 PRINT_FAIL("failed to parse caps %s\n", name);
295 return -EINVAL;
296 }
297 *val |= (1ULL << cap_flag);
298 token = strtok_r(NULL, "|", &saveptr);
299 }
300
301 free(str_cpy);
302 return 0;
303 }
304
parse_retval(const char * str,int * val,const char * name)305 static int parse_retval(const char *str, int *val, const char *name)
306 {
307 /*
308 * INT_MIN is defined as (-INT_MAX -1), i.e. it doesn't expand to a
309 * single int and cannot be parsed with strtol, so we handle it
310 * separately here. In addition, it expands to different expressions in
311 * different compilers so we use a prefixed _INT_MIN instead.
312 */
313 if (strcmp(str, "_INT_MIN") == 0) {
314 *val = INT_MIN;
315 return 0;
316 }
317
318 return parse_int(str, val, name);
319 }
320
update_flags(int * flags,int flag,bool clear)321 static void update_flags(int *flags, int flag, bool clear)
322 {
323 if (clear)
324 *flags &= ~flag;
325 else
326 *flags |= flag;
327 }
328
skip_decl_tag_pfx(const char * s)329 static const char *skip_decl_tag_pfx(const char *s)
330 {
331 int n = 0;
332
333 if (sscanf(s, "comment:%*d:%n", &n) < 0 || !n)
334 return NULL;
335 return s + n;
336 }
337
compare_decl_tags(const void * a,const void * b)338 static int compare_decl_tags(const void *a, const void *b)
339 {
340 return strverscmp(*(const char **)a, *(const char **)b);
341 }
342
343 /*
344 * Compilers don't guarantee order in which BTF attributes would be generated,
345 * while order is important for test tags like __msg.
346 * Each test tag has the following prefix: "comment:" __COUNTER__,
347 * when sorted using strverscmp this gives same order as in the original C code.
348 */
collect_decl_tags(struct btf * btf,int id,int * cnt)349 static const char **collect_decl_tags(struct btf *btf, int id, int *cnt)
350 {
351 const char **tmp, **tags = NULL;
352 const struct btf_type *t;
353 int i;
354
355 *cnt = 0;
356 for (i = 1; i < btf__type_cnt(btf); i++) {
357 t = btf__type_by_id(btf, i);
358 if (!btf_is_decl_tag(t) || t->type != id || btf_decl_tag(t)->component_idx != -1)
359 continue;
360 tmp = realloc(tags, (*cnt + 1) * sizeof(*tags));
361 if (!tmp) {
362 free(tags);
363 return ERR_PTR(-ENOMEM);
364 }
365 tags = tmp;
366 tags[(*cnt)++] = btf__str_by_offset(btf, t->name_off);
367 }
368
369 if (*cnt)
370 qsort(tags, *cnt, sizeof(*tags), compare_decl_tags);
371 return tags;
372 }
373
374 enum arch {
375 ARCH_UNKNOWN = 0x1,
376 ARCH_X86_64 = 0x2,
377 ARCH_ARM64 = 0x4,
378 ARCH_RISCV64 = 0x8,
379 ARCH_S390X = 0x10,
380 ARCH_LOONGARCH = 0x20,
381 };
382
get_current_arch(void)383 static int get_current_arch(void)
384 {
385 #if defined(__x86_64__)
386 return ARCH_X86_64;
387 #elif defined(__aarch64__)
388 return ARCH_ARM64;
389 #elif defined(__riscv) && __riscv_xlen == 64
390 return ARCH_RISCV64;
391 #elif defined(__s390x__)
392 return ARCH_S390X;
393 #elif defined(__loongarch__)
394 return ARCH_LOONGARCH;
395 #endif
396 return ARCH_UNKNOWN;
397 }
398
399 /* Uses btf_decl_tag attributes to describe the expected test
400 * behavior, see bpf_misc.h for detailed description of each attribute
401 * and attribute combinations.
402 */
parse_test_spec(struct test_loader * tester,struct bpf_object * obj,struct bpf_program * prog,struct test_spec * spec)403 static int parse_test_spec(struct test_loader *tester,
404 struct bpf_object *obj,
405 struct bpf_program *prog,
406 struct test_spec *spec)
407 {
408 const char *description = NULL;
409 bool has_unpriv_result = false;
410 bool has_unpriv_retval = false;
411 bool unpriv_xlated_on_next_line = true;
412 bool xlated_on_next_line = true;
413 bool unpriv_jit_on_next_line;
414 bool jit_on_next_line;
415 bool stderr_on_next_line = true;
416 bool unpriv_stderr_on_next_line = true;
417 bool stdout_on_next_line = true;
418 bool unpriv_stdout_on_next_line = true;
419 bool collect_jit = false;
420 const char **tags = NULL;
421 int func_id, i, nr_tags;
422 int err = 0;
423 u32 arch_mask = 0;
424 u32 load_mask = 0;
425 struct btf *btf;
426 enum arch arch;
427
428 memset(spec, 0, sizeof(*spec));
429
430 spec->prog_name = bpf_program__name(prog);
431 spec->prog_flags = testing_prog_flags();
432
433 btf = bpf_object__btf(obj);
434 if (!btf) {
435 ASSERT_FAIL("BPF object has no BTF");
436 return -EINVAL;
437 }
438
439 func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC);
440 if (func_id < 0) {
441 ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name);
442 return -EINVAL;
443 }
444
445 tags = collect_decl_tags(btf, func_id, &nr_tags);
446 if (IS_ERR(tags))
447 return PTR_ERR(tags);
448
449 for (i = 0; i < nr_tags; i++) {
450 const char *s, *val, *msg;
451 bool clear;
452 int flags;
453
454 s = skip_decl_tag_pfx(tags[i]);
455 if (!s)
456 continue;
457 if ((val = str_has_pfx(s, "test_description="))) {
458 description = val;
459 } else if (strcmp(s, "test_expect_failure") == 0) {
460 spec->priv.expect_failure = true;
461 spec->mode_mask |= PRIV;
462 } else if (strcmp(s, "test_expect_success") == 0) {
463 spec->priv.expect_failure = false;
464 spec->mode_mask |= PRIV;
465 } else if (strcmp(s, "test_expect_failure_unpriv") == 0) {
466 spec->unpriv.expect_failure = true;
467 spec->mode_mask |= UNPRIV;
468 has_unpriv_result = true;
469 } else if (strcmp(s, "test_expect_success_unpriv") == 0) {
470 spec->unpriv.expect_failure = false;
471 spec->mode_mask |= UNPRIV;
472 has_unpriv_result = true;
473 } else if (strcmp(s, "test_auxiliary") == 0) {
474 spec->auxiliary = true;
475 spec->mode_mask |= PRIV;
476 } else if (strcmp(s, "test_auxiliary_unpriv") == 0) {
477 spec->auxiliary = true;
478 spec->mode_mask |= UNPRIV;
479 } else if ((msg = str_has_pfx(s, "test_expect_msg="))) {
480 err = push_msg(msg, false, &spec->priv.expect_msgs);
481 if (err)
482 goto cleanup;
483 spec->mode_mask |= PRIV;
484 } else if ((msg = str_has_pfx(s, "test_expect_not_msg="))) {
485 err = push_msg(msg, true, &spec->priv.expect_msgs);
486 if (err)
487 goto cleanup;
488 spec->mode_mask |= PRIV;
489 } else if ((msg = str_has_pfx(s, "test_expect_msg_unpriv="))) {
490 err = push_msg(msg, false, &spec->unpriv.expect_msgs);
491 if (err)
492 goto cleanup;
493 spec->mode_mask |= UNPRIV;
494 } else if ((msg = str_has_pfx(s, "test_expect_not_msg_unpriv="))) {
495 err = push_msg(msg, true, &spec->unpriv.expect_msgs);
496 if (err)
497 goto cleanup;
498 spec->mode_mask |= UNPRIV;
499 } else if ((msg = str_has_pfx(s, "test_jited="))) {
500 if (arch_mask == 0) {
501 PRINT_FAIL("__jited used before __arch_*");
502 goto cleanup;
503 }
504 if (collect_jit) {
505 err = push_disasm_msg(msg, &jit_on_next_line,
506 &spec->priv.jited);
507 if (err)
508 goto cleanup;
509 spec->mode_mask |= PRIV;
510 }
511 } else if ((msg = str_has_pfx(s, "test_jited_unpriv="))) {
512 if (arch_mask == 0) {
513 PRINT_FAIL("__unpriv_jited used before __arch_*");
514 goto cleanup;
515 }
516 if (collect_jit) {
517 err = push_disasm_msg(msg, &unpriv_jit_on_next_line,
518 &spec->unpriv.jited);
519 if (err)
520 goto cleanup;
521 spec->mode_mask |= UNPRIV;
522 }
523 } else if ((msg = str_has_pfx(s, "test_expect_xlated="))) {
524 err = push_disasm_msg(msg, &xlated_on_next_line,
525 &spec->priv.expect_xlated);
526 if (err)
527 goto cleanup;
528 spec->mode_mask |= PRIV;
529 } else if ((msg = str_has_pfx(s, "test_expect_xlated_unpriv="))) {
530 err = push_disasm_msg(msg, &unpriv_xlated_on_next_line,
531 &spec->unpriv.expect_xlated);
532 if (err)
533 goto cleanup;
534 spec->mode_mask |= UNPRIV;
535 } else if ((val = str_has_pfx(s, "test_retval="))) {
536 err = parse_retval(val, &spec->priv.retval, "__retval");
537 if (err)
538 goto cleanup;
539 spec->priv.execute = true;
540 spec->mode_mask |= PRIV;
541 } else if ((val = str_has_pfx(s, "test_retval_unpriv="))) {
542 err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv");
543 if (err)
544 goto cleanup;
545 spec->mode_mask |= UNPRIV;
546 spec->unpriv.execute = true;
547 has_unpriv_retval = true;
548 } else if ((val = str_has_pfx(s, "test_log_level="))) {
549 err = parse_int(val, &spec->log_level, "test log level");
550 if (err)
551 goto cleanup;
552 } else if ((val = str_has_pfx(s, "test_prog_flags="))) {
553 clear = val[0] == '!';
554 if (clear)
555 val++;
556
557 if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) {
558 update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear);
559 } else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) {
560 update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear);
561 } else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) {
562 update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear);
563 } else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) {
564 update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear);
565 } else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) {
566 update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear);
567 } else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) {
568 update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear);
569 } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) {
570 update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear);
571 } else /* assume numeric value */ {
572 err = parse_int(val, &flags, "test prog flags");
573 if (err)
574 goto cleanup;
575 update_flags(&spec->prog_flags, flags, clear);
576 }
577 } else if ((val = str_has_pfx(s, "test_arch="))) {
578 if (strcmp(val, "X86_64") == 0) {
579 arch = ARCH_X86_64;
580 } else if (strcmp(val, "ARM64") == 0) {
581 arch = ARCH_ARM64;
582 } else if (strcmp(val, "RISCV64") == 0) {
583 arch = ARCH_RISCV64;
584 } else if (strcmp(val, "s390x") == 0) {
585 arch = ARCH_S390X;
586 } else if (strcmp(val, "LOONGARCH") == 0) {
587 arch = ARCH_LOONGARCH;
588 } else {
589 PRINT_FAIL("bad arch spec: '%s'\n", val);
590 err = -EINVAL;
591 goto cleanup;
592 }
593 arch_mask |= arch;
594 collect_jit = get_current_arch() == arch;
595 unpriv_jit_on_next_line = true;
596 jit_on_next_line = true;
597 } else if ((val = str_has_pfx(s, "test_btf_path="))) {
598 spec->btf_custom_path = val;
599 } else if ((val = str_has_pfx(s, "test_btf_func_path="))) {
600 spec->btf_custom_func_path = val;
601 } else if ((val = str_has_pfx(s, "test_caps_unpriv="))) {
602 err = parse_caps(val, &spec->unpriv.caps, "test caps");
603 if (err)
604 goto cleanup;
605 spec->mode_mask |= UNPRIV;
606 } else if ((val = str_has_pfx(s, "load_mode="))) {
607 if (strcmp(val, "jited") == 0) {
608 load_mask = JITED;
609 } else if (strcmp(val, "no_jited") == 0) {
610 load_mask = NO_JITED;
611 } else {
612 PRINT_FAIL("bad load spec: '%s'", val);
613 err = -EINVAL;
614 goto cleanup;
615 }
616 } else if ((msg = str_has_pfx(s, "test_expect_stderr="))) {
617 err = push_disasm_msg(msg, &stderr_on_next_line,
618 &spec->priv.stderr);
619 if (err)
620 goto cleanup;
621 } else if ((msg = str_has_pfx(s, "test_expect_stderr_unpriv="))) {
622 err = push_disasm_msg(msg, &unpriv_stderr_on_next_line,
623 &spec->unpriv.stderr);
624 if (err)
625 goto cleanup;
626 } else if ((msg = str_has_pfx(s, "test_expect_stdout="))) {
627 err = push_disasm_msg(msg, &stdout_on_next_line,
628 &spec->priv.stdout);
629 if (err)
630 goto cleanup;
631 } else if ((msg = str_has_pfx(s, "test_expect_stdout_unpriv="))) {
632 err = push_disasm_msg(msg, &unpriv_stdout_on_next_line,
633 &spec->unpriv.stdout);
634 if (err)
635 goto cleanup;
636 } else if ((val = str_has_pfx(s, "test_linear_size="))) {
637 switch (bpf_program__type(prog)) {
638 case BPF_PROG_TYPE_SCHED_ACT:
639 case BPF_PROG_TYPE_SCHED_CLS:
640 case BPF_PROG_TYPE_CGROUP_SKB:
641 err = parse_int(val, &spec->linear_sz, "test linear size");
642 if (err)
643 goto cleanup;
644 break;
645 default:
646 PRINT_FAIL("__linear_size for unsupported program type");
647 err = -EINVAL;
648 goto cleanup;
649 }
650 }
651 }
652
653 spec->arch_mask = arch_mask ?: -1;
654 spec->load_mask = load_mask ?: (JITED | NO_JITED);
655
656 if (spec->mode_mask == 0)
657 spec->mode_mask = PRIV;
658
659 if (spec->mode_mask & PRIV) {
660 spec->priv.name = strdup(spec->prog_name);
661 if (!spec->priv.name) {
662 PRINT_FAIL("failed to allocate memory for priv.name\n");
663 err = -ENOMEM;
664 goto cleanup;
665 }
666
667 if (description) {
668 spec->priv.description = strdup(description);
669 if (!spec->priv.description) {
670 PRINT_FAIL("failed to allocate memory for priv.description\n");
671 err = -ENOMEM;
672 goto cleanup;
673 }
674 }
675 }
676
677 if (spec->mode_mask & UNPRIV) {
678 int name_len = strlen(spec->prog_name);
679 const char *suffix = " @unpriv";
680 int suffix_len = strlen(suffix);
681 char *name;
682
683 name = malloc(name_len + suffix_len + 1);
684 if (!name) {
685 PRINT_FAIL("failed to allocate memory for unpriv.name\n");
686 err = -ENOMEM;
687 goto cleanup;
688 }
689
690 strcpy(name, spec->prog_name);
691 strcpy(&name[name_len], suffix);
692 spec->unpriv.name = name;
693
694 if (description) {
695 int descr_len = strlen(description);
696 char *descr;
697
698 descr = malloc(descr_len + suffix_len + 1);
699 if (!descr) {
700 PRINT_FAIL("failed to allocate memory for unpriv.description\n");
701 err = -ENOMEM;
702 goto cleanup;
703 }
704
705 strcpy(descr, description);
706 strcpy(&descr[descr_len], suffix);
707 spec->unpriv.description = descr;
708 }
709 }
710
711 if (spec->mode_mask & (PRIV | UNPRIV)) {
712 if (!has_unpriv_result)
713 spec->unpriv.expect_failure = spec->priv.expect_failure;
714
715 if (!has_unpriv_retval) {
716 spec->unpriv.retval = spec->priv.retval;
717 spec->unpriv.execute = spec->priv.execute;
718 }
719
720 if (spec->unpriv.expect_msgs.cnt == 0)
721 clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs);
722 if (spec->unpriv.expect_xlated.cnt == 0)
723 clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated);
724 if (spec->unpriv.jited.cnt == 0)
725 clone_msgs(&spec->priv.jited, &spec->unpriv.jited);
726 if (spec->unpriv.stderr.cnt == 0)
727 clone_msgs(&spec->priv.stderr, &spec->unpriv.stderr);
728 if (spec->unpriv.stdout.cnt == 0)
729 clone_msgs(&spec->priv.stdout, &spec->unpriv.stdout);
730 }
731
732 spec->valid = true;
733
734 free(tags);
735 return 0;
736
737 cleanup:
738 free(tags);
739 free_test_spec(spec);
740 return err;
741 }
742
prepare_case(struct test_loader * tester,struct test_spec * spec,struct bpf_object * obj,struct bpf_program * prog)743 static void prepare_case(struct test_loader *tester,
744 struct test_spec *spec,
745 struct bpf_object *obj,
746 struct bpf_program *prog)
747 {
748 int min_log_level = 0, prog_flags;
749
750 if (env.verbosity > VERBOSE_NONE)
751 min_log_level = 1;
752 if (env.verbosity > VERBOSE_VERY)
753 min_log_level = 2;
754
755 bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz);
756
757 /* Make sure we set at least minimal log level, unless test requires
758 * even higher level already. Make sure to preserve independent log
759 * level 4 (verifier stats), though.
760 */
761 if ((spec->log_level & 3) < min_log_level)
762 bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level);
763 else
764 bpf_program__set_log_level(prog, spec->log_level);
765
766 prog_flags = bpf_program__flags(prog);
767 bpf_program__set_flags(prog, prog_flags | spec->prog_flags);
768
769 tester->log_buf[0] = '\0';
770 }
771
emit_verifier_log(const char * log_buf,bool force)772 static void emit_verifier_log(const char *log_buf, bool force)
773 {
774 if (!force && env.verbosity == VERBOSE_NONE)
775 return;
776 fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf);
777 }
778
emit_xlated(const char * xlated,bool force)779 static void emit_xlated(const char *xlated, bool force)
780 {
781 if (!force && env.verbosity == VERBOSE_NONE)
782 return;
783 fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated);
784 }
785
emit_jited(const char * jited,bool force)786 static void emit_jited(const char *jited, bool force)
787 {
788 if (!force && env.verbosity == VERBOSE_NONE)
789 return;
790 fprintf(stdout, "JITED:\n=============\n%s=============\n", jited);
791 }
792
emit_stderr(const char * stderr,bool force)793 static void emit_stderr(const char *stderr, bool force)
794 {
795 if (!force && env.verbosity == VERBOSE_NONE)
796 return;
797 fprintf(stdout, "STDERR:\n=============\n%s=============\n", stderr);
798 }
799
verify_stderr(int prog_fd,struct expected_msgs * msgs)800 static void verify_stderr(int prog_fd, struct expected_msgs *msgs)
801 {
802 LIBBPF_OPTS(bpf_prog_stream_read_opts, ropts);
803 char *buf;
804 int ret;
805
806 if (!msgs->cnt)
807 return;
808
809 buf = malloc(TEST_LOADER_LOG_BUF_SZ);
810 if (!ASSERT_OK_PTR(buf, "malloc"))
811 return;
812
813 ret = bpf_prog_stream_read(prog_fd, 2, buf, TEST_LOADER_LOG_BUF_SZ - 1,
814 &ropts);
815 if (ret > 0) {
816 buf[ret] = '\0';
817 emit_stderr(buf, false);
818 validate_msgs(buf, msgs, emit_stderr);
819 } else {
820 ASSERT_GT(ret, 0, "stderr stream read");
821 }
822
823 free(buf);
824 }
825
verify_test_stderr(struct bpf_object * obj,struct bpf_program * prog)826 void verify_test_stderr(struct bpf_object *obj, struct bpf_program *prog)
827 {
828 struct test_spec spec = {};
829
830 if (parse_test_spec(NULL, obj, prog, &spec))
831 return;
832
833 verify_stderr(bpf_program__fd(prog), &spec.priv.stderr);
834 free_test_spec(&spec);
835 }
836
emit_stdout(const char * bpf_stdout,bool force)837 static void emit_stdout(const char *bpf_stdout, bool force)
838 {
839 if (!force && env.verbosity == VERBOSE_NONE)
840 return;
841 fprintf(stdout, "STDOUT:\n=============\n%s=============\n", bpf_stdout);
842 }
843
match_msg(struct expect_msg * msg,const char ** log)844 static const char *match_msg(struct expect_msg *msg, const char **log)
845 {
846 const char *match = NULL;
847 regmatch_t reg_match[1];
848 int err;
849
850 if (!msg->is_regex) {
851 match = strstr(*log, msg->substr);
852 if (match)
853 *log = match + strlen(msg->substr);
854 } else {
855 err = regexec(&msg->regex, *log, 1, reg_match, 0);
856 if (err == 0) {
857 match = *log + reg_match[0].rm_so;
858 *log += reg_match[0].rm_eo;
859 }
860 }
861 return match;
862 }
863
count_lines(const char * start,const char * end)864 static int count_lines(const char *start, const char *end)
865 {
866 const char *tmp;
867 int n = 0;
868
869 for (tmp = start; tmp < end; ++tmp)
870 if (*tmp == '\n')
871 n++;
872 return n;
873 }
874
875 struct match {
876 const char *start;
877 const char *end;
878 int line;
879 };
880
881 /*
882 * Positive messages are matched sequentially, each next message
883 * is looked for starting from the end of a previous matched one.
884 */
match_positive_msgs(const char * log,struct expected_msgs * msgs,struct match * matches)885 static void match_positive_msgs(const char *log, struct expected_msgs *msgs, struct match *matches)
886 {
887 const char *prev_match;
888 int i, line;
889
890 prev_match = log;
891 line = 0;
892 for (i = 0; i < msgs->cnt; i++) {
893 struct expect_msg *msg = &msgs->patterns[i];
894 const char *match = NULL;
895
896 if (msg->negative)
897 continue;
898
899 match = match_msg(msg, &log);
900 if (match) {
901 line += count_lines(prev_match, match);
902 matches[i].start = match;
903 matches[i].end = log;
904 matches[i].line = line;
905 prev_match = match;
906 }
907 }
908 }
909
910 /*
911 * Each negative messages N located between positive messages P1 and P2
912 * is matched in the span P1.end .. P2.start. Consequently, negative messages
913 * are unordered within the span.
914 */
match_negative_msgs(const char * log,struct expected_msgs * msgs,struct match * matches)915 static void match_negative_msgs(const char *log, struct expected_msgs *msgs, struct match *matches)
916 {
917 const char *start = log, *end, *next, *match;
918 const char *log_end = log + strlen(log);
919 int i, j, next_positive;
920
921 for (i = 0; i < msgs->cnt; i++) {
922 struct expect_msg *msg = &msgs->patterns[i];
923
924 /* positive message bumps span start */
925 if (!msg->negative) {
926 start = matches[i].end ?: start;
927 continue;
928 }
929
930 /* count stride of negative patterns and adjust span end */
931 end = log_end;
932 for (next_positive = i + 1; next_positive < msgs->cnt; next_positive++) {
933 if (!msgs->patterns[next_positive].negative) {
934 end = matches[next_positive].start;
935 break;
936 }
937 }
938
939 /* try matching negative messages within identified span */
940 for (j = i; j < next_positive; j++) {
941 next = start;
942 match = match_msg(msg, &next);
943 if (match && next <= end) {
944 matches[j].start = match;
945 matches[j].end = next;
946 }
947 }
948
949 /* -1 to account for i++ */
950 i = next_positive - 1;
951 }
952 }
953
validate_msgs(const char * log_buf,struct expected_msgs * msgs,void (* emit_fn)(const char * buf,bool force))954 void validate_msgs(const char *log_buf, struct expected_msgs *msgs,
955 void (*emit_fn)(const char *buf, bool force))
956 {
957 struct match matches[msgs->cnt];
958 struct match *prev_match = NULL;
959 int i, j;
960
961 memset(matches, 0, sizeof(*matches) * msgs->cnt);
962 match_positive_msgs(log_buf, msgs, matches);
963 match_negative_msgs(log_buf, msgs, matches);
964
965 for (i = 0; i < msgs->cnt; i++) {
966 struct expect_msg *msg = &msgs->patterns[i];
967 struct match *match = &matches[i];
968 const char *pat_status;
969 bool unexpected;
970 bool wrong_line;
971 bool no_match;
972
973 no_match = !msg->negative && !match->start;
974 wrong_line = !msg->negative &&
975 msg->on_next_line &&
976 prev_match && prev_match->line + 1 != match->line;
977 unexpected = msg->negative && match->start;
978 if (no_match || wrong_line || unexpected) {
979 PRINT_FAIL("expect_msg\n");
980 if (env.verbosity == VERBOSE_NONE)
981 emit_fn(log_buf, true /*force*/);
982 for (j = 0; j <= i; j++) {
983 msg = &msgs->patterns[j];
984 if (j < i)
985 pat_status = "MATCHED ";
986 else if (wrong_line)
987 pat_status = "WRONG LINE";
988 else if (no_match)
989 pat_status = "EXPECTED ";
990 else
991 pat_status = "UNEXPECTED";
992 msg = &msgs->patterns[j];
993 fprintf(stderr, "%s %s: '%s'\n",
994 pat_status,
995 msg->is_regex ? " REGEX" : "SUBSTR",
996 msg->substr);
997 }
998 if (wrong_line) {
999 fprintf(stderr,
1000 "expecting match at line %d, actual match is at line %d\n",
1001 prev_match->line + 1, match->line);
1002 }
1003 break;
1004 }
1005
1006 if (!msg->negative)
1007 prev_match = match;
1008 }
1009 }
1010
1011 struct cap_state {
1012 __u64 old_caps;
1013 bool initialized;
1014 };
1015
drop_capabilities(struct cap_state * caps)1016 static int drop_capabilities(struct cap_state *caps)
1017 {
1018 const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN |
1019 1ULL << CAP_PERFMON | 1ULL << CAP_BPF);
1020 int err;
1021
1022 err = cap_disable_effective(caps_to_drop, &caps->old_caps);
1023 if (err) {
1024 PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(-err));
1025 return err;
1026 }
1027
1028 caps->initialized = true;
1029 return 0;
1030 }
1031
restore_capabilities(struct cap_state * caps)1032 static int restore_capabilities(struct cap_state *caps)
1033 {
1034 int err;
1035
1036 if (!caps->initialized)
1037 return 0;
1038
1039 err = cap_enable_effective(caps->old_caps, NULL);
1040 if (err)
1041 PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(-err));
1042 caps->initialized = false;
1043 return err;
1044 }
1045
can_execute_unpriv(struct test_loader * tester,struct test_spec * spec)1046 static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec)
1047 {
1048 if (sysctl_unpriv_disabled < 0)
1049 sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0;
1050 if (sysctl_unpriv_disabled)
1051 return false;
1052 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS)
1053 return false;
1054 return true;
1055 }
1056
is_unpriv_capable_map(struct bpf_map * map)1057 static bool is_unpriv_capable_map(struct bpf_map *map)
1058 {
1059 enum bpf_map_type type;
1060 __u32 flags;
1061
1062 type = bpf_map__type(map);
1063
1064 switch (type) {
1065 case BPF_MAP_TYPE_HASH:
1066 case BPF_MAP_TYPE_PERCPU_HASH:
1067 case BPF_MAP_TYPE_HASH_OF_MAPS:
1068 flags = bpf_map__map_flags(map);
1069 return !(flags & BPF_F_ZERO_SEED);
1070 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
1071 case BPF_MAP_TYPE_ARRAY:
1072 case BPF_MAP_TYPE_RINGBUF:
1073 case BPF_MAP_TYPE_PROG_ARRAY:
1074 case BPF_MAP_TYPE_CGROUP_ARRAY:
1075 case BPF_MAP_TYPE_PERCPU_ARRAY:
1076 case BPF_MAP_TYPE_USER_RINGBUF:
1077 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
1078 case BPF_MAP_TYPE_CGROUP_STORAGE:
1079 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
1080 return true;
1081 default:
1082 return false;
1083 }
1084 }
1085
do_prog_test_run(int fd_prog,int * retval,bool empty_opts,int linear_sz)1086 static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts, int linear_sz)
1087 {
1088 __u8 tmp_out[TEST_DATA_LEN << 2] = {};
1089 __u8 tmp_in[TEST_DATA_LEN] = {};
1090 struct __sk_buff ctx = {};
1091 int err, saved_errno;
1092 LIBBPF_OPTS(bpf_test_run_opts, topts,
1093 .data_in = tmp_in,
1094 .data_size_in = sizeof(tmp_in),
1095 .data_out = tmp_out,
1096 .data_size_out = sizeof(tmp_out),
1097 .repeat = 1,
1098 );
1099
1100 if (linear_sz) {
1101 ctx.data_end = linear_sz;
1102 topts.ctx_in = &ctx;
1103 topts.ctx_size_in = sizeof(ctx);
1104 }
1105
1106 if (empty_opts) {
1107 memset(&topts, 0, sizeof(struct bpf_test_run_opts));
1108 topts.sz = sizeof(struct bpf_test_run_opts);
1109 }
1110 err = bpf_prog_test_run_opts(fd_prog, &topts);
1111 saved_errno = errno;
1112
1113 if (err) {
1114 PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ",
1115 saved_errno, strerror(saved_errno));
1116 return err;
1117 }
1118
1119 ASSERT_OK(0, "bpf_prog_test_run");
1120 *retval = topts.retval;
1121
1122 return 0;
1123 }
1124
should_do_test_run(struct test_spec * spec,struct test_subspec * subspec)1125 static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec)
1126 {
1127 if (!subspec->execute)
1128 return false;
1129
1130 if (subspec->expect_failure)
1131 return false;
1132
1133 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) {
1134 if (env.verbosity != VERBOSE_NONE)
1135 printf("alignment prevents execution\n");
1136 return false;
1137 }
1138
1139 return true;
1140 }
1141
1142 /* Get a disassembly of BPF program after verifier applies all rewrites */
get_xlated_program_text(int prog_fd,char * text,size_t text_sz)1143 static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz)
1144 {
1145 struct bpf_insn *insn_start = NULL, *insn, *insn_end;
1146 __u32 insns_cnt = 0, i;
1147 char buf[64];
1148 FILE *out = NULL;
1149 int err;
1150
1151 err = get_xlated_program(prog_fd, &insn_start, &insns_cnt);
1152 if (!ASSERT_OK(err, "get_xlated_program"))
1153 goto out;
1154 out = fmemopen(text, text_sz, "w");
1155 if (!ASSERT_OK_PTR(out, "open_memstream"))
1156 goto out;
1157 insn_end = insn_start + insns_cnt;
1158 insn = insn_start;
1159 while (insn < insn_end) {
1160 i = insn - insn_start;
1161 insn = disasm_insn(insn, buf, sizeof(buf));
1162 fprintf(out, "%d: %s\n", i, buf);
1163 }
1164 fflush(out);
1165
1166 out:
1167 free(insn_start);
1168 if (out)
1169 fclose(out);
1170 return err;
1171 }
1172
1173 /* Read the bpf stream corresponding to the stream_id */
get_stream(int stream_id,int prog_fd,char * text,size_t text_sz)1174 static int get_stream(int stream_id, int prog_fd, char *text, size_t text_sz)
1175 {
1176 LIBBPF_OPTS(bpf_prog_stream_read_opts, ropts);
1177 int ret;
1178
1179 ret = bpf_prog_stream_read(prog_fd, stream_id, text, text_sz, &ropts);
1180 ASSERT_GT(ret, 0, "stream read");
1181 text[ret] = '\0';
1182
1183 return ret;
1184 }
1185
1186 /*
1187 * Fix up the program's BTF using BTF from a separate file.
1188 *
1189 * For __naked subprogs, clang drops parameter names from BTF. Find FUNC
1190 * entries with anonymous parameters and replace their FUNC_PROTO with the
1191 * properly-named version from the custom file.
1192 */
fixup_btf_from_path(struct bpf_object * obj,const char * path)1193 static int fixup_btf_from_path(struct bpf_object *obj, const char *path)
1194 {
1195 struct btf *prog_btf, *custom_btf;
1196 __u32 i, j, cnt, custom_cnt;
1197 int err = 0;
1198
1199 prog_btf = bpf_object__btf(obj);
1200 if (!prog_btf)
1201 return 0;
1202
1203 custom_btf = btf__parse(path, NULL);
1204 if (!ASSERT_OK_PTR(custom_btf, "parse_custom_btf"))
1205 return -EINVAL;
1206
1207 cnt = btf__type_cnt(prog_btf);
1208 custom_cnt = btf__type_cnt(custom_btf);
1209
1210 /* Fix up FUNC entries with anonymous params.
1211 * Save all data from prog_btf BEFORE calling btf__add_*,
1212 * since those calls may reallocate the BTF data buffer
1213 * and invalidate any pointers obtained from btf__type_by_id.
1214 */
1215 for (i = 1; i < cnt; i++) {
1216 const struct btf_type *t = btf__type_by_id(prog_btf, i);
1217 const struct btf_type *fp, *custom_t, *custom_fp;
1218 const struct btf_param *params, *custom_params;
1219 __u32 ret_type_id, vlen;
1220 __u32 *prog_param_types = NULL;
1221 const char *name;
1222 int new_proto_id;
1223
1224 if (!btf_is_func(t))
1225 continue;
1226
1227 fp = btf__type_by_id(prog_btf, t->type);
1228 if (!fp || !btf_is_func_proto(fp) || btf_vlen(fp) == 0)
1229 continue;
1230
1231 /* Check if any param is anonymous */
1232 params = btf_params(fp);
1233 if (params[0].name_off != 0)
1234 continue;
1235
1236 /* Find matching FUNC by name in custom BTF */
1237 name = btf__name_by_offset(prog_btf, t->name_off);
1238 if (!name)
1239 continue;
1240
1241 for (j = 1; j < custom_cnt; j++) {
1242 const char *cname;
1243
1244 custom_t = btf__type_by_id(custom_btf, j);
1245 if (!btf_is_func(custom_t))
1246 continue;
1247 cname = btf__name_by_offset(custom_btf, custom_t->name_off);
1248 if (cname && strcmp(name, cname) == 0)
1249 break;
1250 }
1251 if (j >= custom_cnt)
1252 continue;
1253
1254 custom_fp = btf__type_by_id(custom_btf, custom_t->type);
1255 if (!custom_fp || !btf_is_func_proto(custom_fp))
1256 continue;
1257
1258 vlen = btf_vlen(fp);
1259 if (vlen != btf_vlen(custom_fp))
1260 continue;
1261
1262 /* Save data before btf__add_* calls invalidate pointers */
1263 ret_type_id = fp->type;
1264 prog_param_types = malloc(vlen * sizeof(*prog_param_types));
1265 if (!prog_param_types) {
1266 err = -ENOMEM;
1267 break;
1268 }
1269 for (j = 0; j < vlen; j++)
1270 prog_param_types[j] = params[j].type;
1271
1272 /* Add a new FUNC_PROTO: param names from custom, types from prog */
1273 new_proto_id = btf__add_func_proto(prog_btf, ret_type_id);
1274 if (new_proto_id < 0) {
1275 err = new_proto_id;
1276 free(prog_param_types);
1277 break;
1278 }
1279
1280 custom_params = btf_params(custom_fp);
1281 for (j = 0; j < vlen; j++) {
1282 const char *pname;
1283
1284 pname = btf__name_by_offset(custom_btf, custom_params[j].name_off);
1285 err = btf__add_func_param(prog_btf, pname ?: "", prog_param_types[j]);
1286 if (err)
1287 break;
1288 }
1289 free(prog_param_types);
1290 if (err)
1291 break;
1292
1293 /* Update the FUNC to point to the new FUNC_PROTO (re-fetch
1294 * since btf__add_* may have reallocated the data buffer).
1295 */
1296 ((struct btf_type *)btf__type_by_id(prog_btf, i))->type = new_proto_id;
1297 }
1298
1299 btf__free(custom_btf);
1300 return err;
1301 }
1302
1303 /* this function is forced noinline and has short generic name to look better
1304 * in test_progs output (in case of a failure)
1305 */
1306 static noinline
run_subtest(struct test_loader * tester,struct bpf_object_open_opts * open_opts,const void * obj_bytes,size_t obj_byte_cnt,struct test_spec * specs,struct test_spec * spec,bool unpriv)1307 void run_subtest(struct test_loader *tester,
1308 struct bpf_object_open_opts *open_opts,
1309 const void *obj_bytes,
1310 size_t obj_byte_cnt,
1311 struct test_spec *specs,
1312 struct test_spec *spec,
1313 bool unpriv)
1314 {
1315 struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv;
1316 int current_runtime = is_jit_enabled() ? JITED : NO_JITED;
1317 struct bpf_program *tprog = NULL, *tprog_iter;
1318 struct bpf_link *link, *links[32] = {};
1319 struct test_spec *spec_iter;
1320 struct cap_state caps = {};
1321 struct bpf_object *tobj;
1322 struct bpf_map *map;
1323 int retval, err, i;
1324 int links_cnt = 0;
1325 bool should_load;
1326
1327 if (!test__start_subtest_with_desc(subspec->name, subspec->description))
1328 return;
1329
1330 if ((get_current_arch() & spec->arch_mask) == 0) {
1331 test__skip();
1332 return;
1333 }
1334
1335 if ((current_runtime & spec->load_mask) == 0) {
1336 test__skip();
1337 return;
1338 }
1339
1340 if (unpriv) {
1341 if (!can_execute_unpriv(tester, spec)) {
1342 test__skip();
1343 test__end_subtest();
1344 return;
1345 }
1346 if (drop_capabilities(&caps)) {
1347 test__end_subtest();
1348 return;
1349 }
1350 if (subspec->caps) {
1351 err = cap_enable_effective(subspec->caps, NULL);
1352 if (err) {
1353 PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err));
1354 goto subtest_cleanup;
1355 }
1356 }
1357 }
1358
1359 /* Implicitly reset to NULL if next test case doesn't specify.
1360 * btf_custom_func_path also serves as btf_custom_path for kfunc resolution.
1361 */
1362 open_opts->btf_custom_path = spec->btf_custom_path;
1363 if (!open_opts->btf_custom_path)
1364 open_opts->btf_custom_path = spec->btf_custom_func_path;
1365
1366 tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts);
1367 if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */
1368 goto subtest_cleanup;
1369
1370 /* Fix up __naked subprog BTF using a separate file with named params */
1371 if (spec->btf_custom_func_path) {
1372 err = fixup_btf_from_path(tobj, spec->btf_custom_func_path);
1373 if (err) {
1374 PRINT_FAIL("failed to fixup BTF from %s: %d\n",
1375 spec->btf_custom_func_path, err);
1376 goto tobj_cleanup;
1377 }
1378 }
1379
1380 i = 0;
1381 bpf_object__for_each_program(tprog_iter, tobj) {
1382 spec_iter = &specs[i++];
1383 should_load = false;
1384
1385 if (spec_iter->valid) {
1386 if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) {
1387 tprog = tprog_iter;
1388 should_load = true;
1389 }
1390
1391 if (spec_iter->auxiliary &&
1392 spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV))
1393 should_load = true;
1394 }
1395
1396 bpf_program__set_autoload(tprog_iter, should_load);
1397 }
1398
1399 prepare_case(tester, spec, tobj, tprog);
1400
1401 /* By default bpf_object__load() automatically creates all
1402 * maps declared in the skeleton. Some map types are only
1403 * allowed in priv mode. Disable autoload for such maps in
1404 * unpriv mode.
1405 */
1406 bpf_object__for_each_map(map, tobj)
1407 bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map));
1408
1409 err = bpf_object__load(tobj);
1410 if (subspec->expect_failure) {
1411 if (!ASSERT_ERR(err, "unexpected_load_success")) {
1412 emit_verifier_log(tester->log_buf, false /*force*/);
1413 goto tobj_cleanup;
1414 }
1415 } else {
1416 if (!ASSERT_OK(err, "unexpected_load_failure")) {
1417 emit_verifier_log(tester->log_buf, true /*force*/);
1418 goto tobj_cleanup;
1419 }
1420 }
1421 emit_verifier_log(tester->log_buf, false /*force*/);
1422 validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log);
1423
1424 /* Restore capabilities because the kernel will silently ignore requests
1425 * for program info (such as xlated program text) if we are not
1426 * bpf-capable. Also, for some reason test_verifier executes programs
1427 * with all capabilities restored. Do the same here.
1428 */
1429 if (restore_capabilities(&caps))
1430 goto tobj_cleanup;
1431
1432 if (subspec->expect_xlated.cnt) {
1433 err = get_xlated_program_text(bpf_program__fd(tprog),
1434 tester->log_buf, tester->log_buf_sz);
1435 if (err)
1436 goto tobj_cleanup;
1437 emit_xlated(tester->log_buf, false /*force*/);
1438 validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated);
1439 }
1440
1441 if (subspec->jited.cnt) {
1442 err = get_jited_program_text(bpf_program__fd(tprog),
1443 tester->log_buf, tester->log_buf_sz);
1444 if (err == -EOPNOTSUPP) {
1445 printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__);
1446 printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__);
1447 test__skip();
1448 goto tobj_cleanup;
1449 }
1450 if (!ASSERT_EQ(err, 0, "get_jited_program_text"))
1451 goto tobj_cleanup;
1452 emit_jited(tester->log_buf, false /*force*/);
1453 validate_msgs(tester->log_buf, &subspec->jited, emit_jited);
1454 }
1455
1456 if (should_do_test_run(spec, subspec)) {
1457 /* Do bpf_map__attach_struct_ops() for each struct_ops map.
1458 * This should trigger bpf_struct_ops->reg callback on kernel side.
1459 */
1460 bpf_object__for_each_map(map, tobj) {
1461 if (!bpf_map__autocreate(map) ||
1462 bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS)
1463 continue;
1464 if (links_cnt >= ARRAY_SIZE(links)) {
1465 PRINT_FAIL("too many struct_ops maps");
1466 goto tobj_cleanup;
1467 }
1468 link = bpf_map__attach_struct_ops(map);
1469 if (!link) {
1470 PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n",
1471 bpf_map__name(map), -errno);
1472 goto tobj_cleanup;
1473 }
1474 links[links_cnt++] = link;
1475 }
1476
1477 if (tester->pre_execution_cb) {
1478 err = tester->pre_execution_cb(tobj);
1479 if (err) {
1480 PRINT_FAIL("pre_execution_cb failed: %d\n", err);
1481 goto tobj_cleanup;
1482 }
1483 }
1484
1485 err = do_prog_test_run(bpf_program__fd(tprog), &retval,
1486 bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false,
1487 spec->linear_sz);
1488 if (!err && retval != subspec->retval && subspec->retval != POINTER_VALUE) {
1489 PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval);
1490 goto tobj_cleanup;
1491 }
1492
1493 verify_stderr(bpf_program__fd(tprog), &subspec->stderr);
1494
1495 if (subspec->stdout.cnt) {
1496 err = get_stream(1, bpf_program__fd(tprog),
1497 tester->log_buf, tester->log_buf_sz);
1498 if (err <= 0) {
1499 PRINT_FAIL("Unexpected retval from get_stream(): %d, errno = %d\n",
1500 err, errno);
1501 goto tobj_cleanup;
1502 }
1503 emit_stdout(tester->log_buf, false /*force*/);
1504 validate_msgs(tester->log_buf, &subspec->stdout, emit_stdout);
1505 }
1506
1507 /* redo bpf_map__attach_struct_ops for each test */
1508 while (links_cnt > 0)
1509 bpf_link__destroy(links[--links_cnt]);
1510 }
1511
1512 tobj_cleanup:
1513 while (links_cnt > 0)
1514 bpf_link__destroy(links[--links_cnt]);
1515 bpf_object__close(tobj);
1516 subtest_cleanup:
1517 test__end_subtest();
1518 restore_capabilities(&caps);
1519 }
1520
process_subtest(struct test_loader * tester,const char * skel_name,skel_elf_bytes_fn elf_bytes_factory)1521 static void process_subtest(struct test_loader *tester,
1522 const char *skel_name,
1523 skel_elf_bytes_fn elf_bytes_factory)
1524 {
1525 LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name);
1526 struct test_spec *specs = NULL;
1527 struct bpf_object *obj = NULL;
1528 struct bpf_program *prog;
1529 const void *obj_bytes;
1530 int err, i, nr_progs;
1531 size_t obj_byte_cnt;
1532
1533 if (tester_init(tester) < 0)
1534 return; /* failed to initialize tester */
1535
1536 obj_bytes = elf_bytes_factory(&obj_byte_cnt);
1537 obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts);
1538 if (!ASSERT_OK_PTR(obj, "obj_open_mem"))
1539 return;
1540
1541 nr_progs = 0;
1542 bpf_object__for_each_program(prog, obj)
1543 ++nr_progs;
1544
1545 specs = calloc(nr_progs, sizeof(struct test_spec));
1546 if (!ASSERT_OK_PTR(specs, "specs_alloc"))
1547 return;
1548
1549 i = 0;
1550 bpf_object__for_each_program(prog, obj) {
1551 /* ignore tests for which we can't derive test specification */
1552 err = parse_test_spec(tester, obj, prog, &specs[i++]);
1553 if (err)
1554 PRINT_FAIL("Can't parse test spec for program '%s'\n",
1555 bpf_program__name(prog));
1556 }
1557
1558 i = 0;
1559 bpf_object__for_each_program(prog, obj) {
1560 struct test_spec *spec = &specs[i++];
1561
1562 if (!spec->valid || spec->auxiliary)
1563 continue;
1564
1565 if (spec->mode_mask & PRIV)
1566 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
1567 specs, spec, false);
1568 if (spec->mode_mask & UNPRIV)
1569 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
1570 specs, spec, true);
1571
1572 }
1573
1574 for (i = 0; i < nr_progs; ++i)
1575 free_test_spec(&specs[i]);
1576 free(specs);
1577 bpf_object__close(obj);
1578 }
1579
test_loader__run_subtests(struct test_loader * tester,const char * skel_name,skel_elf_bytes_fn elf_bytes_factory)1580 void test_loader__run_subtests(struct test_loader *tester,
1581 const char *skel_name,
1582 skel_elf_bytes_fn elf_bytes_factory)
1583 {
1584 /* see comment in run_subtest() for why we do this function nesting */
1585 process_subtest(tester, skel_name, elf_bytes_factory);
1586 }
1587