1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3 #include <linux/capability.h>
4 #include <stdlib.h>
5 #include <regex.h>
6 #include <test_progs.h>
7 #include <bpf/btf.h>
8
9 #include "autoconf_helper.h"
10 #include "disasm_helpers.h"
11 #include "unpriv_helpers.h"
12 #include "cap_helpers.h"
13 #include "jit_disasm_helpers.h"
14
15 #define str_has_pfx(str, pfx) \
16 (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0)
17
18 #define TEST_LOADER_LOG_BUF_SZ 2097152
19
20 #define TEST_TAG_EXPECT_FAILURE "comment:test_expect_failure"
21 #define TEST_TAG_EXPECT_SUCCESS "comment:test_expect_success"
22 #define TEST_TAG_EXPECT_MSG_PFX "comment:test_expect_msg="
23 #define TEST_TAG_EXPECT_XLATED_PFX "comment:test_expect_xlated="
24 #define TEST_TAG_EXPECT_FAILURE_UNPRIV "comment:test_expect_failure_unpriv"
25 #define TEST_TAG_EXPECT_SUCCESS_UNPRIV "comment:test_expect_success_unpriv"
26 #define TEST_TAG_EXPECT_MSG_PFX_UNPRIV "comment:test_expect_msg_unpriv="
27 #define TEST_TAG_EXPECT_XLATED_PFX_UNPRIV "comment:test_expect_xlated_unpriv="
28 #define TEST_TAG_LOG_LEVEL_PFX "comment:test_log_level="
29 #define TEST_TAG_PROG_FLAGS_PFX "comment:test_prog_flags="
30 #define TEST_TAG_DESCRIPTION_PFX "comment:test_description="
31 #define TEST_TAG_RETVAL_PFX "comment:test_retval="
32 #define TEST_TAG_RETVAL_PFX_UNPRIV "comment:test_retval_unpriv="
33 #define TEST_TAG_AUXILIARY "comment:test_auxiliary"
34 #define TEST_TAG_AUXILIARY_UNPRIV "comment:test_auxiliary_unpriv"
35 #define TEST_BTF_PATH "comment:test_btf_path="
36 #define TEST_TAG_ARCH "comment:test_arch="
37 #define TEST_TAG_JITED_PFX "comment:test_jited="
38 #define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv="
39 #define TEST_TAG_CAPS_UNPRIV "comment:test_caps_unpriv="
40 #define TEST_TAG_LOAD_MODE_PFX "comment:load_mode="
41
42 /* Warning: duplicated in bpf_misc.h */
43 #define POINTER_VALUE 0xbadcafe
44 #define TEST_DATA_LEN 64
45
46 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
47 #define EFFICIENT_UNALIGNED_ACCESS 1
48 #else
49 #define EFFICIENT_UNALIGNED_ACCESS 0
50 #endif
51
52 static int sysctl_unpriv_disabled = -1;
53
54 enum mode {
55 PRIV = 1,
56 UNPRIV = 2
57 };
58
59 enum load_mode {
60 JITED = 1 << 0,
61 NO_JITED = 1 << 1,
62 };
63
64 struct expect_msg {
65 const char *substr; /* substring match */
66 regex_t regex;
67 bool is_regex;
68 bool on_next_line;
69 };
70
71 struct expected_msgs {
72 struct expect_msg *patterns;
73 size_t cnt;
74 };
75
76 struct test_subspec {
77 char *name;
78 bool expect_failure;
79 struct expected_msgs expect_msgs;
80 struct expected_msgs expect_xlated;
81 struct expected_msgs jited;
82 int retval;
83 bool execute;
84 __u64 caps;
85 };
86
87 struct test_spec {
88 const char *prog_name;
89 struct test_subspec priv;
90 struct test_subspec unpriv;
91 const char *btf_custom_path;
92 int log_level;
93 int prog_flags;
94 int mode_mask;
95 int arch_mask;
96 int load_mask;
97 bool auxiliary;
98 bool valid;
99 };
100
tester_init(struct test_loader * tester)101 static int tester_init(struct test_loader *tester)
102 {
103 if (!tester->log_buf) {
104 tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ;
105 tester->log_buf = calloc(tester->log_buf_sz, 1);
106 if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf"))
107 return -ENOMEM;
108 }
109
110 return 0;
111 }
112
test_loader_fini(struct test_loader * tester)113 void test_loader_fini(struct test_loader *tester)
114 {
115 if (!tester)
116 return;
117
118 free(tester->log_buf);
119 }
120
free_msgs(struct expected_msgs * msgs)121 static void free_msgs(struct expected_msgs *msgs)
122 {
123 int i;
124
125 for (i = 0; i < msgs->cnt; i++)
126 if (msgs->patterns[i].is_regex)
127 regfree(&msgs->patterns[i].regex);
128 free(msgs->patterns);
129 msgs->patterns = NULL;
130 msgs->cnt = 0;
131 }
132
free_test_spec(struct test_spec * spec)133 static void free_test_spec(struct test_spec *spec)
134 {
135 /* Deallocate expect_msgs arrays. */
136 free_msgs(&spec->priv.expect_msgs);
137 free_msgs(&spec->unpriv.expect_msgs);
138 free_msgs(&spec->priv.expect_xlated);
139 free_msgs(&spec->unpriv.expect_xlated);
140 free_msgs(&spec->priv.jited);
141 free_msgs(&spec->unpriv.jited);
142
143 free(spec->priv.name);
144 free(spec->unpriv.name);
145 spec->priv.name = NULL;
146 spec->unpriv.name = NULL;
147 }
148
149 /* Compiles regular expression matching pattern.
150 * Pattern has a special syntax:
151 *
152 * pattern := (<verbatim text> | regex)*
153 * regex := "{{" <posix extended regular expression> "}}"
154 *
155 * In other words, pattern is a verbatim text with inclusion
156 * of regular expressions enclosed in "{{" "}}" pairs.
157 * For example, pattern "foo{{[0-9]+}}" matches strings like
158 * "foo0", "foo007", etc.
159 */
compile_regex(const char * pattern,regex_t * regex)160 static int compile_regex(const char *pattern, regex_t *regex)
161 {
162 char err_buf[256], buf[256] = {}, *ptr, *buf_end;
163 const char *original_pattern = pattern;
164 bool in_regex = false;
165 int err;
166
167 buf_end = buf + sizeof(buf);
168 ptr = buf;
169 while (*pattern && ptr < buf_end - 2) {
170 if (!in_regex && str_has_pfx(pattern, "{{")) {
171 in_regex = true;
172 pattern += 2;
173 continue;
174 }
175 if (in_regex && str_has_pfx(pattern, "}}")) {
176 in_regex = false;
177 pattern += 2;
178 continue;
179 }
180 if (in_regex) {
181 *ptr++ = *pattern++;
182 continue;
183 }
184 /* list of characters that need escaping for extended posix regex */
185 if (strchr(".[]\\()*+?{}|^$", *pattern)) {
186 *ptr++ = '\\';
187 *ptr++ = *pattern++;
188 continue;
189 }
190 *ptr++ = *pattern++;
191 }
192 if (*pattern) {
193 PRINT_FAIL("Regexp too long: '%s'\n", original_pattern);
194 return -EINVAL;
195 }
196 if (in_regex) {
197 PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern);
198 return -EINVAL;
199 }
200 err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE);
201 if (err != 0) {
202 regerror(err, regex, err_buf, sizeof(err_buf));
203 PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf);
204 return -EINVAL;
205 }
206 return 0;
207 }
208
__push_msg(const char * pattern,bool on_next_line,struct expected_msgs * msgs)209 static int __push_msg(const char *pattern, bool on_next_line, struct expected_msgs *msgs)
210 {
211 struct expect_msg *msg;
212 void *tmp;
213 int err;
214
215 tmp = realloc(msgs->patterns,
216 (1 + msgs->cnt) * sizeof(struct expect_msg));
217 if (!tmp) {
218 ASSERT_FAIL("failed to realloc memory for messages\n");
219 return -ENOMEM;
220 }
221 msgs->patterns = tmp;
222 msg = &msgs->patterns[msgs->cnt];
223 msg->on_next_line = on_next_line;
224 msg->substr = pattern;
225 msg->is_regex = false;
226 if (strstr(pattern, "{{")) {
227 err = compile_regex(pattern, &msg->regex);
228 if (err)
229 return err;
230 msg->is_regex = true;
231 }
232 msgs->cnt += 1;
233 return 0;
234 }
235
clone_msgs(struct expected_msgs * from,struct expected_msgs * to)236 static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to)
237 {
238 struct expect_msg *msg;
239 int i, err;
240
241 for (i = 0; i < from->cnt; i++) {
242 msg = &from->patterns[i];
243 err = __push_msg(msg->substr, msg->on_next_line, to);
244 if (err)
245 return err;
246 }
247 return 0;
248 }
249
push_msg(const char * substr,struct expected_msgs * msgs)250 static int push_msg(const char *substr, struct expected_msgs *msgs)
251 {
252 return __push_msg(substr, false, msgs);
253 }
254
push_disasm_msg(const char * regex_str,bool * on_next_line,struct expected_msgs * msgs)255 static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs)
256 {
257 int err;
258
259 if (strcmp(regex_str, "...") == 0) {
260 *on_next_line = false;
261 return 0;
262 }
263 err = __push_msg(regex_str, *on_next_line, msgs);
264 if (err)
265 return err;
266 *on_next_line = true;
267 return 0;
268 }
269
parse_int(const char * str,int * val,const char * name)270 static int parse_int(const char *str, int *val, const char *name)
271 {
272 char *end;
273 long tmp;
274
275 errno = 0;
276 if (str_has_pfx(str, "0x"))
277 tmp = strtol(str + 2, &end, 16);
278 else
279 tmp = strtol(str, &end, 10);
280 if (errno || end[0] != '\0') {
281 PRINT_FAIL("failed to parse %s from '%s'\n", name, str);
282 return -EINVAL;
283 }
284 *val = tmp;
285 return 0;
286 }
287
parse_caps(const char * str,__u64 * val,const char * name)288 static int parse_caps(const char *str, __u64 *val, const char *name)
289 {
290 int cap_flag = 0;
291 char *token = NULL, *saveptr = NULL;
292
293 char *str_cpy = strdup(str);
294 if (str_cpy == NULL) {
295 PRINT_FAIL("Memory allocation failed\n");
296 return -EINVAL;
297 }
298
299 token = strtok_r(str_cpy, "|", &saveptr);
300 while (token != NULL) {
301 errno = 0;
302 if (!strncmp("CAP_", token, sizeof("CAP_") - 1)) {
303 PRINT_FAIL("define %s constant in bpf_misc.h, failed to parse caps\n", token);
304 return -EINVAL;
305 }
306 cap_flag = strtol(token, NULL, 10);
307 if (!cap_flag || errno) {
308 PRINT_FAIL("failed to parse caps %s\n", name);
309 return -EINVAL;
310 }
311 *val |= (1ULL << cap_flag);
312 token = strtok_r(NULL, "|", &saveptr);
313 }
314
315 free(str_cpy);
316 return 0;
317 }
318
parse_retval(const char * str,int * val,const char * name)319 static int parse_retval(const char *str, int *val, const char *name)
320 {
321 /*
322 * INT_MIN is defined as (-INT_MAX -1), i.e. it doesn't expand to a
323 * single int and cannot be parsed with strtol, so we handle it
324 * separately here. In addition, it expands to different expressions in
325 * different compilers so we use a prefixed _INT_MIN instead.
326 */
327 if (strcmp(str, "_INT_MIN") == 0) {
328 *val = INT_MIN;
329 return 0;
330 }
331
332 return parse_int(str, val, name);
333 }
334
update_flags(int * flags,int flag,bool clear)335 static void update_flags(int *flags, int flag, bool clear)
336 {
337 if (clear)
338 *flags &= ~flag;
339 else
340 *flags |= flag;
341 }
342
343 /* Matches a string of form '<pfx>[^=]=.*' and returns it's suffix.
344 * Used to parse btf_decl_tag values.
345 * Such values require unique prefix because compiler does not add
346 * same __attribute__((btf_decl_tag(...))) twice.
347 * Test suite uses two-component tags for such cases:
348 *
349 * <pfx> __COUNTER__ '='
350 *
351 * For example, two consecutive __msg tags '__msg("foo") __msg("foo")'
352 * would be encoded as:
353 *
354 * [18] DECL_TAG 'comment:test_expect_msg=0=foo' type_id=15 component_idx=-1
355 * [19] DECL_TAG 'comment:test_expect_msg=1=foo' type_id=15 component_idx=-1
356 *
357 * And the purpose of this function is to extract 'foo' from the above.
358 */
skip_dynamic_pfx(const char * s,const char * pfx)359 static const char *skip_dynamic_pfx(const char *s, const char *pfx)
360 {
361 const char *msg;
362
363 if (strncmp(s, pfx, strlen(pfx)) != 0)
364 return NULL;
365 msg = s + strlen(pfx);
366 msg = strchr(msg, '=');
367 if (!msg)
368 return NULL;
369 return msg + 1;
370 }
371
372 enum arch {
373 ARCH_UNKNOWN = 0x1,
374 ARCH_X86_64 = 0x2,
375 ARCH_ARM64 = 0x4,
376 ARCH_RISCV64 = 0x8,
377 };
378
get_current_arch(void)379 static int get_current_arch(void)
380 {
381 #if defined(__x86_64__)
382 return ARCH_X86_64;
383 #elif defined(__aarch64__)
384 return ARCH_ARM64;
385 #elif defined(__riscv) && __riscv_xlen == 64
386 return ARCH_RISCV64;
387 #endif
388 return ARCH_UNKNOWN;
389 }
390
391 /* Uses btf_decl_tag attributes to describe the expected test
392 * behavior, see bpf_misc.h for detailed description of each attribute
393 * and attribute combinations.
394 */
parse_test_spec(struct test_loader * tester,struct bpf_object * obj,struct bpf_program * prog,struct test_spec * spec)395 static int parse_test_spec(struct test_loader *tester,
396 struct bpf_object *obj,
397 struct bpf_program *prog,
398 struct test_spec *spec)
399 {
400 const char *description = NULL;
401 bool has_unpriv_result = false;
402 bool has_unpriv_retval = false;
403 bool unpriv_xlated_on_next_line = true;
404 bool xlated_on_next_line = true;
405 bool unpriv_jit_on_next_line;
406 bool jit_on_next_line;
407 bool collect_jit = false;
408 int func_id, i, err = 0;
409 u32 arch_mask = 0;
410 u32 load_mask = 0;
411 struct btf *btf;
412 enum arch arch;
413
414 memset(spec, 0, sizeof(*spec));
415
416 spec->prog_name = bpf_program__name(prog);
417 spec->prog_flags = testing_prog_flags();
418
419 btf = bpf_object__btf(obj);
420 if (!btf) {
421 ASSERT_FAIL("BPF object has no BTF");
422 return -EINVAL;
423 }
424
425 func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC);
426 if (func_id < 0) {
427 ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name);
428 return -EINVAL;
429 }
430
431 for (i = 1; i < btf__type_cnt(btf); i++) {
432 const char *s, *val, *msg;
433 const struct btf_type *t;
434 bool clear;
435 int flags;
436
437 t = btf__type_by_id(btf, i);
438 if (!btf_is_decl_tag(t))
439 continue;
440
441 if (t->type != func_id || btf_decl_tag(t)->component_idx != -1)
442 continue;
443
444 s = btf__str_by_offset(btf, t->name_off);
445 if (str_has_pfx(s, TEST_TAG_DESCRIPTION_PFX)) {
446 description = s + sizeof(TEST_TAG_DESCRIPTION_PFX) - 1;
447 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE) == 0) {
448 spec->priv.expect_failure = true;
449 spec->mode_mask |= PRIV;
450 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS) == 0) {
451 spec->priv.expect_failure = false;
452 spec->mode_mask |= PRIV;
453 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE_UNPRIV) == 0) {
454 spec->unpriv.expect_failure = true;
455 spec->mode_mask |= UNPRIV;
456 has_unpriv_result = true;
457 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS_UNPRIV) == 0) {
458 spec->unpriv.expect_failure = false;
459 spec->mode_mask |= UNPRIV;
460 has_unpriv_result = true;
461 } else if (strcmp(s, TEST_TAG_AUXILIARY) == 0) {
462 spec->auxiliary = true;
463 spec->mode_mask |= PRIV;
464 } else if (strcmp(s, TEST_TAG_AUXILIARY_UNPRIV) == 0) {
465 spec->auxiliary = true;
466 spec->mode_mask |= UNPRIV;
467 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX))) {
468 err = push_msg(msg, &spec->priv.expect_msgs);
469 if (err)
470 goto cleanup;
471 spec->mode_mask |= PRIV;
472 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX_UNPRIV))) {
473 err = push_msg(msg, &spec->unpriv.expect_msgs);
474 if (err)
475 goto cleanup;
476 spec->mode_mask |= UNPRIV;
477 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX))) {
478 if (arch_mask == 0) {
479 PRINT_FAIL("__jited used before __arch_*");
480 goto cleanup;
481 }
482 if (collect_jit) {
483 err = push_disasm_msg(msg, &jit_on_next_line,
484 &spec->priv.jited);
485 if (err)
486 goto cleanup;
487 spec->mode_mask |= PRIV;
488 }
489 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX_UNPRIV))) {
490 if (arch_mask == 0) {
491 PRINT_FAIL("__unpriv_jited used before __arch_*");
492 goto cleanup;
493 }
494 if (collect_jit) {
495 err = push_disasm_msg(msg, &unpriv_jit_on_next_line,
496 &spec->unpriv.jited);
497 if (err)
498 goto cleanup;
499 spec->mode_mask |= UNPRIV;
500 }
501 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX))) {
502 err = push_disasm_msg(msg, &xlated_on_next_line,
503 &spec->priv.expect_xlated);
504 if (err)
505 goto cleanup;
506 spec->mode_mask |= PRIV;
507 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX_UNPRIV))) {
508 err = push_disasm_msg(msg, &unpriv_xlated_on_next_line,
509 &spec->unpriv.expect_xlated);
510 if (err)
511 goto cleanup;
512 spec->mode_mask |= UNPRIV;
513 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX)) {
514 val = s + sizeof(TEST_TAG_RETVAL_PFX) - 1;
515 err = parse_retval(val, &spec->priv.retval, "__retval");
516 if (err)
517 goto cleanup;
518 spec->priv.execute = true;
519 spec->mode_mask |= PRIV;
520 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX_UNPRIV)) {
521 val = s + sizeof(TEST_TAG_RETVAL_PFX_UNPRIV) - 1;
522 err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv");
523 if (err)
524 goto cleanup;
525 spec->mode_mask |= UNPRIV;
526 spec->unpriv.execute = true;
527 has_unpriv_retval = true;
528 } else if (str_has_pfx(s, TEST_TAG_LOG_LEVEL_PFX)) {
529 val = s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1;
530 err = parse_int(val, &spec->log_level, "test log level");
531 if (err)
532 goto cleanup;
533 } else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) {
534 val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1;
535
536 clear = val[0] == '!';
537 if (clear)
538 val++;
539
540 if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) {
541 update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear);
542 } else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) {
543 update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear);
544 } else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) {
545 update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear);
546 } else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) {
547 update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear);
548 } else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) {
549 update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear);
550 } else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) {
551 update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear);
552 } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) {
553 update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear);
554 } else /* assume numeric value */ {
555 err = parse_int(val, &flags, "test prog flags");
556 if (err)
557 goto cleanup;
558 update_flags(&spec->prog_flags, flags, clear);
559 }
560 } else if (str_has_pfx(s, TEST_TAG_ARCH)) {
561 val = s + sizeof(TEST_TAG_ARCH) - 1;
562 if (strcmp(val, "X86_64") == 0) {
563 arch = ARCH_X86_64;
564 } else if (strcmp(val, "ARM64") == 0) {
565 arch = ARCH_ARM64;
566 } else if (strcmp(val, "RISCV64") == 0) {
567 arch = ARCH_RISCV64;
568 } else {
569 PRINT_FAIL("bad arch spec: '%s'", val);
570 err = -EINVAL;
571 goto cleanup;
572 }
573 arch_mask |= arch;
574 collect_jit = get_current_arch() == arch;
575 unpriv_jit_on_next_line = true;
576 jit_on_next_line = true;
577 } else if (str_has_pfx(s, TEST_BTF_PATH)) {
578 spec->btf_custom_path = s + sizeof(TEST_BTF_PATH) - 1;
579 } else if (str_has_pfx(s, TEST_TAG_CAPS_UNPRIV)) {
580 val = s + sizeof(TEST_TAG_CAPS_UNPRIV) - 1;
581 err = parse_caps(val, &spec->unpriv.caps, "test caps");
582 if (err)
583 goto cleanup;
584 spec->mode_mask |= UNPRIV;
585 } else if (str_has_pfx(s, TEST_TAG_LOAD_MODE_PFX)) {
586 val = s + sizeof(TEST_TAG_LOAD_MODE_PFX) - 1;
587 if (strcmp(val, "jited") == 0) {
588 load_mask = JITED;
589 } else if (strcmp(val, "no_jited") == 0) {
590 load_mask = NO_JITED;
591 } else {
592 PRINT_FAIL("bad load spec: '%s'", val);
593 err = -EINVAL;
594 goto cleanup;
595 }
596 }
597 }
598
599 spec->arch_mask = arch_mask ?: -1;
600 spec->load_mask = load_mask ?: (JITED | NO_JITED);
601
602 if (spec->mode_mask == 0)
603 spec->mode_mask = PRIV;
604
605 if (!description)
606 description = spec->prog_name;
607
608 if (spec->mode_mask & PRIV) {
609 spec->priv.name = strdup(description);
610 if (!spec->priv.name) {
611 PRINT_FAIL("failed to allocate memory for priv.name\n");
612 err = -ENOMEM;
613 goto cleanup;
614 }
615 }
616
617 if (spec->mode_mask & UNPRIV) {
618 int descr_len = strlen(description);
619 const char *suffix = " @unpriv";
620 char *name;
621
622 name = malloc(descr_len + strlen(suffix) + 1);
623 if (!name) {
624 PRINT_FAIL("failed to allocate memory for unpriv.name\n");
625 err = -ENOMEM;
626 goto cleanup;
627 }
628
629 strcpy(name, description);
630 strcpy(&name[descr_len], suffix);
631 spec->unpriv.name = name;
632 }
633
634 if (spec->mode_mask & (PRIV | UNPRIV)) {
635 if (!has_unpriv_result)
636 spec->unpriv.expect_failure = spec->priv.expect_failure;
637
638 if (!has_unpriv_retval) {
639 spec->unpriv.retval = spec->priv.retval;
640 spec->unpriv.execute = spec->priv.execute;
641 }
642
643 if (spec->unpriv.expect_msgs.cnt == 0)
644 clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs);
645 if (spec->unpriv.expect_xlated.cnt == 0)
646 clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated);
647 if (spec->unpriv.jited.cnt == 0)
648 clone_msgs(&spec->priv.jited, &spec->unpriv.jited);
649 }
650
651 spec->valid = true;
652
653 return 0;
654
655 cleanup:
656 free_test_spec(spec);
657 return err;
658 }
659
prepare_case(struct test_loader * tester,struct test_spec * spec,struct bpf_object * obj,struct bpf_program * prog)660 static void prepare_case(struct test_loader *tester,
661 struct test_spec *spec,
662 struct bpf_object *obj,
663 struct bpf_program *prog)
664 {
665 int min_log_level = 0, prog_flags;
666
667 if (env.verbosity > VERBOSE_NONE)
668 min_log_level = 1;
669 if (env.verbosity > VERBOSE_VERY)
670 min_log_level = 2;
671
672 bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz);
673
674 /* Make sure we set at least minimal log level, unless test requires
675 * even higher level already. Make sure to preserve independent log
676 * level 4 (verifier stats), though.
677 */
678 if ((spec->log_level & 3) < min_log_level)
679 bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level);
680 else
681 bpf_program__set_log_level(prog, spec->log_level);
682
683 prog_flags = bpf_program__flags(prog);
684 bpf_program__set_flags(prog, prog_flags | spec->prog_flags);
685
686 tester->log_buf[0] = '\0';
687 }
688
emit_verifier_log(const char * log_buf,bool force)689 static void emit_verifier_log(const char *log_buf, bool force)
690 {
691 if (!force && env.verbosity == VERBOSE_NONE)
692 return;
693 fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf);
694 }
695
emit_xlated(const char * xlated,bool force)696 static void emit_xlated(const char *xlated, bool force)
697 {
698 if (!force && env.verbosity == VERBOSE_NONE)
699 return;
700 fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated);
701 }
702
emit_jited(const char * jited,bool force)703 static void emit_jited(const char *jited, bool force)
704 {
705 if (!force && env.verbosity == VERBOSE_NONE)
706 return;
707 fprintf(stdout, "JITED:\n=============\n%s=============\n", jited);
708 }
709
validate_msgs(char * log_buf,struct expected_msgs * msgs,void (* emit_fn)(const char * buf,bool force))710 static void validate_msgs(char *log_buf, struct expected_msgs *msgs,
711 void (*emit_fn)(const char *buf, bool force))
712 {
713 const char *log = log_buf, *prev_match;
714 regmatch_t reg_match[1];
715 int prev_match_line;
716 int match_line;
717 int i, j, err;
718
719 prev_match_line = -1;
720 match_line = 0;
721 prev_match = log;
722 for (i = 0; i < msgs->cnt; i++) {
723 struct expect_msg *msg = &msgs->patterns[i];
724 const char *match = NULL, *pat_status;
725 bool wrong_line = false;
726
727 if (!msg->is_regex) {
728 match = strstr(log, msg->substr);
729 if (match)
730 log = match + strlen(msg->substr);
731 } else {
732 err = regexec(&msg->regex, log, 1, reg_match, 0);
733 if (err == 0) {
734 match = log + reg_match[0].rm_so;
735 log += reg_match[0].rm_eo;
736 }
737 }
738
739 if (match) {
740 for (; prev_match < match; ++prev_match)
741 if (*prev_match == '\n')
742 ++match_line;
743 wrong_line = msg->on_next_line && prev_match_line >= 0 &&
744 prev_match_line + 1 != match_line;
745 }
746
747 if (!match || wrong_line) {
748 PRINT_FAIL("expect_msg\n");
749 if (env.verbosity == VERBOSE_NONE)
750 emit_fn(log_buf, true /*force*/);
751 for (j = 0; j <= i; j++) {
752 msg = &msgs->patterns[j];
753 if (j < i)
754 pat_status = "MATCHED ";
755 else if (wrong_line)
756 pat_status = "WRONG LINE";
757 else
758 pat_status = "EXPECTED ";
759 msg = &msgs->patterns[j];
760 fprintf(stderr, "%s %s: '%s'\n",
761 pat_status,
762 msg->is_regex ? " REGEX" : "SUBSTR",
763 msg->substr);
764 }
765 if (wrong_line) {
766 fprintf(stderr,
767 "expecting match at line %d, actual match is at line %d\n",
768 prev_match_line + 1, match_line);
769 }
770 break;
771 }
772
773 prev_match_line = match_line;
774 }
775 }
776
777 struct cap_state {
778 __u64 old_caps;
779 bool initialized;
780 };
781
drop_capabilities(struct cap_state * caps)782 static int drop_capabilities(struct cap_state *caps)
783 {
784 const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN |
785 1ULL << CAP_PERFMON | 1ULL << CAP_BPF);
786 int err;
787
788 err = cap_disable_effective(caps_to_drop, &caps->old_caps);
789 if (err) {
790 PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(-err));
791 return err;
792 }
793
794 caps->initialized = true;
795 return 0;
796 }
797
restore_capabilities(struct cap_state * caps)798 static int restore_capabilities(struct cap_state *caps)
799 {
800 int err;
801
802 if (!caps->initialized)
803 return 0;
804
805 err = cap_enable_effective(caps->old_caps, NULL);
806 if (err)
807 PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(-err));
808 caps->initialized = false;
809 return err;
810 }
811
can_execute_unpriv(struct test_loader * tester,struct test_spec * spec)812 static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec)
813 {
814 if (sysctl_unpriv_disabled < 0)
815 sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0;
816 if (sysctl_unpriv_disabled)
817 return false;
818 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS)
819 return false;
820 return true;
821 }
822
is_unpriv_capable_map(struct bpf_map * map)823 static bool is_unpriv_capable_map(struct bpf_map *map)
824 {
825 enum bpf_map_type type;
826 __u32 flags;
827
828 type = bpf_map__type(map);
829
830 switch (type) {
831 case BPF_MAP_TYPE_HASH:
832 case BPF_MAP_TYPE_PERCPU_HASH:
833 case BPF_MAP_TYPE_HASH_OF_MAPS:
834 flags = bpf_map__map_flags(map);
835 return !(flags & BPF_F_ZERO_SEED);
836 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
837 case BPF_MAP_TYPE_ARRAY:
838 case BPF_MAP_TYPE_RINGBUF:
839 case BPF_MAP_TYPE_PROG_ARRAY:
840 case BPF_MAP_TYPE_CGROUP_ARRAY:
841 case BPF_MAP_TYPE_PERCPU_ARRAY:
842 case BPF_MAP_TYPE_USER_RINGBUF:
843 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
844 case BPF_MAP_TYPE_CGROUP_STORAGE:
845 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
846 return true;
847 default:
848 return false;
849 }
850 }
851
do_prog_test_run(int fd_prog,int * retval,bool empty_opts)852 static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts)
853 {
854 __u8 tmp_out[TEST_DATA_LEN << 2] = {};
855 __u8 tmp_in[TEST_DATA_LEN] = {};
856 int err, saved_errno;
857 LIBBPF_OPTS(bpf_test_run_opts, topts,
858 .data_in = tmp_in,
859 .data_size_in = sizeof(tmp_in),
860 .data_out = tmp_out,
861 .data_size_out = sizeof(tmp_out),
862 .repeat = 1,
863 );
864
865 if (empty_opts) {
866 memset(&topts, 0, sizeof(struct bpf_test_run_opts));
867 topts.sz = sizeof(struct bpf_test_run_opts);
868 }
869 err = bpf_prog_test_run_opts(fd_prog, &topts);
870 saved_errno = errno;
871
872 if (err) {
873 PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ",
874 saved_errno, strerror(saved_errno));
875 return err;
876 }
877
878 ASSERT_OK(0, "bpf_prog_test_run");
879 *retval = topts.retval;
880
881 return 0;
882 }
883
should_do_test_run(struct test_spec * spec,struct test_subspec * subspec)884 static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec)
885 {
886 if (!subspec->execute)
887 return false;
888
889 if (subspec->expect_failure)
890 return false;
891
892 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) {
893 if (env.verbosity != VERBOSE_NONE)
894 printf("alignment prevents execution\n");
895 return false;
896 }
897
898 return true;
899 }
900
901 /* Get a disassembly of BPF program after verifier applies all rewrites */
get_xlated_program_text(int prog_fd,char * text,size_t text_sz)902 static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz)
903 {
904 struct bpf_insn *insn_start = NULL, *insn, *insn_end;
905 __u32 insns_cnt = 0, i;
906 char buf[64];
907 FILE *out = NULL;
908 int err;
909
910 err = get_xlated_program(prog_fd, &insn_start, &insns_cnt);
911 if (!ASSERT_OK(err, "get_xlated_program"))
912 goto out;
913 out = fmemopen(text, text_sz, "w");
914 if (!ASSERT_OK_PTR(out, "open_memstream"))
915 goto out;
916 insn_end = insn_start + insns_cnt;
917 insn = insn_start;
918 while (insn < insn_end) {
919 i = insn - insn_start;
920 insn = disasm_insn(insn, buf, sizeof(buf));
921 fprintf(out, "%d: %s\n", i, buf);
922 }
923 fflush(out);
924
925 out:
926 free(insn_start);
927 if (out)
928 fclose(out);
929 return err;
930 }
931
932 /* this function is forced noinline and has short generic name to look better
933 * in test_progs output (in case of a failure)
934 */
935 static noinline
run_subtest(struct test_loader * tester,struct bpf_object_open_opts * open_opts,const void * obj_bytes,size_t obj_byte_cnt,struct test_spec * specs,struct test_spec * spec,bool unpriv)936 void run_subtest(struct test_loader *tester,
937 struct bpf_object_open_opts *open_opts,
938 const void *obj_bytes,
939 size_t obj_byte_cnt,
940 struct test_spec *specs,
941 struct test_spec *spec,
942 bool unpriv)
943 {
944 struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv;
945 int current_runtime = is_jit_enabled() ? JITED : NO_JITED;
946 struct bpf_program *tprog = NULL, *tprog_iter;
947 struct bpf_link *link, *links[32] = {};
948 struct test_spec *spec_iter;
949 struct cap_state caps = {};
950 struct bpf_object *tobj;
951 struct bpf_map *map;
952 int retval, err, i;
953 int links_cnt = 0;
954 bool should_load;
955
956 if (!test__start_subtest(subspec->name))
957 return;
958
959 if ((get_current_arch() & spec->arch_mask) == 0) {
960 test__skip();
961 return;
962 }
963
964 if ((current_runtime & spec->load_mask) == 0) {
965 test__skip();
966 return;
967 }
968
969 if (unpriv) {
970 if (!can_execute_unpriv(tester, spec)) {
971 test__skip();
972 test__end_subtest();
973 return;
974 }
975 if (drop_capabilities(&caps)) {
976 test__end_subtest();
977 return;
978 }
979 if (subspec->caps) {
980 err = cap_enable_effective(subspec->caps, NULL);
981 if (err) {
982 PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err));
983 goto subtest_cleanup;
984 }
985 }
986 }
987
988 /* Implicitly reset to NULL if next test case doesn't specify */
989 open_opts->btf_custom_path = spec->btf_custom_path;
990
991 tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts);
992 if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */
993 goto subtest_cleanup;
994
995 i = 0;
996 bpf_object__for_each_program(tprog_iter, tobj) {
997 spec_iter = &specs[i++];
998 should_load = false;
999
1000 if (spec_iter->valid) {
1001 if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) {
1002 tprog = tprog_iter;
1003 should_load = true;
1004 }
1005
1006 if (spec_iter->auxiliary &&
1007 spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV))
1008 should_load = true;
1009 }
1010
1011 bpf_program__set_autoload(tprog_iter, should_load);
1012 }
1013
1014 prepare_case(tester, spec, tobj, tprog);
1015
1016 /* By default bpf_object__load() automatically creates all
1017 * maps declared in the skeleton. Some map types are only
1018 * allowed in priv mode. Disable autoload for such maps in
1019 * unpriv mode.
1020 */
1021 bpf_object__for_each_map(map, tobj)
1022 bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map));
1023
1024 err = bpf_object__load(tobj);
1025 if (subspec->expect_failure) {
1026 if (!ASSERT_ERR(err, "unexpected_load_success")) {
1027 emit_verifier_log(tester->log_buf, false /*force*/);
1028 goto tobj_cleanup;
1029 }
1030 } else {
1031 if (!ASSERT_OK(err, "unexpected_load_failure")) {
1032 emit_verifier_log(tester->log_buf, true /*force*/);
1033 goto tobj_cleanup;
1034 }
1035 }
1036 emit_verifier_log(tester->log_buf, false /*force*/);
1037 validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log);
1038
1039 /* Restore capabilities because the kernel will silently ignore requests
1040 * for program info (such as xlated program text) if we are not
1041 * bpf-capable. Also, for some reason test_verifier executes programs
1042 * with all capabilities restored. Do the same here.
1043 */
1044 if (restore_capabilities(&caps))
1045 goto tobj_cleanup;
1046
1047 if (subspec->expect_xlated.cnt) {
1048 err = get_xlated_program_text(bpf_program__fd(tprog),
1049 tester->log_buf, tester->log_buf_sz);
1050 if (err)
1051 goto tobj_cleanup;
1052 emit_xlated(tester->log_buf, false /*force*/);
1053 validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated);
1054 }
1055
1056 if (subspec->jited.cnt) {
1057 err = get_jited_program_text(bpf_program__fd(tprog),
1058 tester->log_buf, tester->log_buf_sz);
1059 if (err == -EOPNOTSUPP) {
1060 printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__);
1061 printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__);
1062 test__skip();
1063 goto tobj_cleanup;
1064 }
1065 if (!ASSERT_EQ(err, 0, "get_jited_program_text"))
1066 goto tobj_cleanup;
1067 emit_jited(tester->log_buf, false /*force*/);
1068 validate_msgs(tester->log_buf, &subspec->jited, emit_jited);
1069 }
1070
1071 if (should_do_test_run(spec, subspec)) {
1072 /* Do bpf_map__attach_struct_ops() for each struct_ops map.
1073 * This should trigger bpf_struct_ops->reg callback on kernel side.
1074 */
1075 bpf_object__for_each_map(map, tobj) {
1076 if (!bpf_map__autocreate(map) ||
1077 bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS)
1078 continue;
1079 if (links_cnt >= ARRAY_SIZE(links)) {
1080 PRINT_FAIL("too many struct_ops maps");
1081 goto tobj_cleanup;
1082 }
1083 link = bpf_map__attach_struct_ops(map);
1084 if (!link) {
1085 PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n",
1086 bpf_map__name(map), err);
1087 goto tobj_cleanup;
1088 }
1089 links[links_cnt++] = link;
1090 }
1091
1092 if (tester->pre_execution_cb) {
1093 err = tester->pre_execution_cb(tobj);
1094 if (err) {
1095 PRINT_FAIL("pre_execution_cb failed: %d\n", err);
1096 goto tobj_cleanup;
1097 }
1098 }
1099
1100 err = do_prog_test_run(bpf_program__fd(tprog), &retval,
1101 bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false);
1102 if (!err && retval != subspec->retval && subspec->retval != POINTER_VALUE) {
1103 PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval);
1104 goto tobj_cleanup;
1105 }
1106 /* redo bpf_map__attach_struct_ops for each test */
1107 while (links_cnt > 0)
1108 bpf_link__destroy(links[--links_cnt]);
1109 }
1110
1111 tobj_cleanup:
1112 while (links_cnt > 0)
1113 bpf_link__destroy(links[--links_cnt]);
1114 bpf_object__close(tobj);
1115 subtest_cleanup:
1116 test__end_subtest();
1117 restore_capabilities(&caps);
1118 }
1119
process_subtest(struct test_loader * tester,const char * skel_name,skel_elf_bytes_fn elf_bytes_factory)1120 static void process_subtest(struct test_loader *tester,
1121 const char *skel_name,
1122 skel_elf_bytes_fn elf_bytes_factory)
1123 {
1124 LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name);
1125 struct test_spec *specs = NULL;
1126 struct bpf_object *obj = NULL;
1127 struct bpf_program *prog;
1128 const void *obj_bytes;
1129 int err, i, nr_progs;
1130 size_t obj_byte_cnt;
1131
1132 if (tester_init(tester) < 0)
1133 return; /* failed to initialize tester */
1134
1135 obj_bytes = elf_bytes_factory(&obj_byte_cnt);
1136 obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts);
1137 if (!ASSERT_OK_PTR(obj, "obj_open_mem"))
1138 return;
1139
1140 nr_progs = 0;
1141 bpf_object__for_each_program(prog, obj)
1142 ++nr_progs;
1143
1144 specs = calloc(nr_progs, sizeof(struct test_spec));
1145 if (!ASSERT_OK_PTR(specs, "specs_alloc"))
1146 return;
1147
1148 i = 0;
1149 bpf_object__for_each_program(prog, obj) {
1150 /* ignore tests for which we can't derive test specification */
1151 err = parse_test_spec(tester, obj, prog, &specs[i++]);
1152 if (err)
1153 PRINT_FAIL("Can't parse test spec for program '%s'\n",
1154 bpf_program__name(prog));
1155 }
1156
1157 i = 0;
1158 bpf_object__for_each_program(prog, obj) {
1159 struct test_spec *spec = &specs[i++];
1160
1161 if (!spec->valid || spec->auxiliary)
1162 continue;
1163
1164 if (spec->mode_mask & PRIV)
1165 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
1166 specs, spec, false);
1167 if (spec->mode_mask & UNPRIV)
1168 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt,
1169 specs, spec, true);
1170
1171 }
1172
1173 for (i = 0; i < nr_progs; ++i)
1174 free_test_spec(&specs[i]);
1175 free(specs);
1176 bpf_object__close(obj);
1177 }
1178
test_loader__run_subtests(struct test_loader * tester,const char * skel_name,skel_elf_bytes_fn elf_bytes_factory)1179 void test_loader__run_subtests(struct test_loader *tester,
1180 const char *skel_name,
1181 skel_elf_bytes_fn elf_bytes_factory)
1182 {
1183 /* see comment in run_subtest() for why we do this function nesting */
1184 process_subtest(tester, skel_name, elf_bytes_factory);
1185 }
1186