1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/capability.h> 4 #include <linux/err.h> 5 #include <stdlib.h> 6 #include <test_progs.h> 7 #include <bpf/btf.h> 8 9 #include "autoconf_helper.h" 10 #include "disasm_helpers.h" 11 #include "unpriv_helpers.h" 12 #include "cap_helpers.h" 13 #include "jit_disasm_helpers.h" 14 15 static inline const char *str_has_pfx(const char *str, const char *pfx) 16 { 17 size_t len = strlen(pfx); 18 19 return strncmp(str, pfx, len) == 0 ? str + len : NULL; 20 } 21 22 #define TEST_LOADER_LOG_BUF_SZ 2097152 23 24 25 /* Warning: duplicated in bpf_misc.h */ 26 #define POINTER_VALUE 0xbadcafe 27 #define TEST_DATA_LEN 64 28 29 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 30 #define EFFICIENT_UNALIGNED_ACCESS 1 31 #else 32 #define EFFICIENT_UNALIGNED_ACCESS 0 33 #endif 34 35 static int sysctl_unpriv_disabled = -1; 36 37 enum mode { 38 PRIV = 1, 39 UNPRIV = 2 40 }; 41 42 enum load_mode { 43 JITED = 1 << 0, 44 NO_JITED = 1 << 1, 45 }; 46 47 struct test_subspec { 48 char *name; 49 char *description; 50 bool expect_failure; 51 struct expected_msgs expect_msgs; 52 struct expected_msgs expect_xlated; 53 struct expected_msgs jited; 54 struct expected_msgs stderr; 55 struct expected_msgs stdout; 56 int retval; 57 bool execute; 58 __u64 caps; 59 }; 60 61 struct test_spec { 62 const char *prog_name; 63 struct test_subspec priv; 64 struct test_subspec unpriv; 65 const char *btf_custom_path; 66 const char *btf_custom_func_path; 67 int log_level; 68 int prog_flags; 69 int mode_mask; 70 int arch_mask; 71 int load_mask; 72 int linear_sz; 73 bool auxiliary; 74 bool valid; 75 }; 76 77 static int tester_init(struct test_loader *tester) 78 { 79 if (!tester->log_buf) { 80 tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ; 81 tester->log_buf = calloc(tester->log_buf_sz, 1); 82 if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf")) 83 return -ENOMEM; 84 } 85 86 return 0; 87 } 88 89 void test_loader_fini(struct test_loader *tester) 90 { 91 if (!tester) 92 return; 93 94 free(tester->log_buf); 95 } 96 97 void free_msgs(struct expected_msgs *msgs) 98 { 99 int i; 100 101 for (i = 0; i < msgs->cnt; i++) 102 if (msgs->patterns[i].is_regex) 103 regfree(&msgs->patterns[i].regex); 104 free(msgs->patterns); 105 msgs->patterns = NULL; 106 msgs->cnt = 0; 107 } 108 109 static void free_test_spec(struct test_spec *spec) 110 { 111 /* Deallocate expect_msgs arrays. */ 112 free_msgs(&spec->priv.expect_msgs); 113 free_msgs(&spec->unpriv.expect_msgs); 114 free_msgs(&spec->priv.expect_xlated); 115 free_msgs(&spec->unpriv.expect_xlated); 116 free_msgs(&spec->priv.jited); 117 free_msgs(&spec->unpriv.jited); 118 free_msgs(&spec->unpriv.stderr); 119 free_msgs(&spec->priv.stderr); 120 free_msgs(&spec->unpriv.stdout); 121 free_msgs(&spec->priv.stdout); 122 123 free(spec->priv.name); 124 free(spec->priv.description); 125 free(spec->unpriv.name); 126 free(spec->unpriv.description); 127 spec->priv.name = NULL; 128 spec->priv.description = NULL; 129 spec->unpriv.name = NULL; 130 spec->unpriv.description = NULL; 131 } 132 133 /* Compiles regular expression matching pattern. 134 * Pattern has a special syntax: 135 * 136 * pattern := (<verbatim text> | regex)* 137 * regex := "{{" <posix extended regular expression> "}}" 138 * 139 * In other words, pattern is a verbatim text with inclusion 140 * of regular expressions enclosed in "{{" "}}" pairs. 141 * For example, pattern "foo{{[0-9]+}}" matches strings like 142 * "foo0", "foo007", etc. 143 */ 144 static int compile_regex(const char *pattern, regex_t *regex) 145 { 146 char err_buf[256], buf[256] = {}, *ptr, *buf_end; 147 const char *original_pattern = pattern, *next; 148 bool in_regex = false; 149 int err; 150 151 buf_end = buf + sizeof(buf); 152 ptr = buf; 153 while (*pattern && ptr < buf_end - 2) { 154 if (!in_regex && (next = str_has_pfx(pattern, "{{"))) { 155 in_regex = true; 156 pattern = next; 157 continue; 158 } 159 if (in_regex && (next = str_has_pfx(pattern, "}}"))) { 160 in_regex = false; 161 pattern = next; 162 continue; 163 } 164 if (in_regex) { 165 *ptr++ = *pattern++; 166 continue; 167 } 168 /* list of characters that need escaping for extended posix regex */ 169 if (strchr(".[]\\()*+?{}|^$", *pattern)) { 170 *ptr++ = '\\'; 171 *ptr++ = *pattern++; 172 continue; 173 } 174 *ptr++ = *pattern++; 175 } 176 if (*pattern) { 177 PRINT_FAIL("Regexp too long: '%s'\n", original_pattern); 178 return -EINVAL; 179 } 180 if (in_regex) { 181 PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern); 182 return -EINVAL; 183 } 184 err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE); 185 if (err != 0) { 186 regerror(err, regex, err_buf, sizeof(err_buf)); 187 PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf); 188 return -EINVAL; 189 } 190 return 0; 191 } 192 193 static int __push_msg(const char *pattern, bool on_next_line, bool negative, 194 struct expected_msgs *msgs) 195 { 196 struct expect_msg *msg; 197 void *tmp; 198 int err; 199 200 tmp = realloc(msgs->patterns, 201 (1 + msgs->cnt) * sizeof(struct expect_msg)); 202 if (!tmp) { 203 ASSERT_FAIL("failed to realloc memory for messages\n"); 204 return -ENOMEM; 205 } 206 msgs->patterns = tmp; 207 msg = &msgs->patterns[msgs->cnt]; 208 msg->on_next_line = on_next_line; 209 msg->substr = pattern; 210 msg->negative = negative; 211 msg->is_regex = false; 212 if (strstr(pattern, "{{")) { 213 err = compile_regex(pattern, &msg->regex); 214 if (err) 215 return err; 216 msg->is_regex = true; 217 } 218 msgs->cnt += 1; 219 return 0; 220 } 221 222 static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to) 223 { 224 struct expect_msg *msg; 225 int i, err; 226 227 for (i = 0; i < from->cnt; i++) { 228 msg = &from->patterns[i]; 229 err = __push_msg(msg->substr, msg->on_next_line, msg->negative, to); 230 if (err) 231 return err; 232 } 233 return 0; 234 } 235 236 static int push_msg(const char *substr, bool negative, struct expected_msgs *msgs) 237 { 238 return __push_msg(substr, false, negative, msgs); 239 } 240 241 static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs) 242 { 243 int err; 244 245 if (strcmp(regex_str, "...") == 0) { 246 *on_next_line = false; 247 return 0; 248 } 249 err = __push_msg(regex_str, *on_next_line, false, msgs); 250 if (err) 251 return err; 252 *on_next_line = true; 253 return 0; 254 } 255 256 static int parse_int(const char *str, int *val, const char *name) 257 { 258 char *end; 259 long tmp; 260 261 errno = 0; 262 if (str_has_pfx(str, "0x")) 263 tmp = strtol(str + 2, &end, 16); 264 else 265 tmp = strtol(str, &end, 10); 266 if (errno || end[0] != '\0') { 267 PRINT_FAIL("failed to parse %s from '%s'\n", name, str); 268 return -EINVAL; 269 } 270 *val = tmp; 271 return 0; 272 } 273 274 static int parse_caps(const char *str, __u64 *val, const char *name) 275 { 276 int cap_flag = 0; 277 char *token = NULL, *saveptr = NULL; 278 279 char *str_cpy = strdup(str); 280 if (str_cpy == NULL) { 281 PRINT_FAIL("Memory allocation failed\n"); 282 return -EINVAL; 283 } 284 285 token = strtok_r(str_cpy, "|", &saveptr); 286 while (token != NULL) { 287 errno = 0; 288 if (!strncmp("CAP_", token, sizeof("CAP_") - 1)) { 289 PRINT_FAIL("define %s constant in bpf_misc.h, failed to parse caps\n", token); 290 return -EINVAL; 291 } 292 cap_flag = strtol(token, NULL, 10); 293 if (!cap_flag || errno) { 294 PRINT_FAIL("failed to parse caps %s\n", name); 295 return -EINVAL; 296 } 297 *val |= (1ULL << cap_flag); 298 token = strtok_r(NULL, "|", &saveptr); 299 } 300 301 free(str_cpy); 302 return 0; 303 } 304 305 static int parse_retval(const char *str, int *val, const char *name) 306 { 307 /* 308 * INT_MIN is defined as (-INT_MAX -1), i.e. it doesn't expand to a 309 * single int and cannot be parsed with strtol, so we handle it 310 * separately here. In addition, it expands to different expressions in 311 * different compilers so we use a prefixed _INT_MIN instead. 312 */ 313 if (strcmp(str, "_INT_MIN") == 0) { 314 *val = INT_MIN; 315 return 0; 316 } 317 318 return parse_int(str, val, name); 319 } 320 321 static void update_flags(int *flags, int flag, bool clear) 322 { 323 if (clear) 324 *flags &= ~flag; 325 else 326 *flags |= flag; 327 } 328 329 static const char *skip_decl_tag_pfx(const char *s) 330 { 331 int n = 0; 332 333 if (sscanf(s, "comment:%*d:%n", &n) < 0 || !n) 334 return NULL; 335 return s + n; 336 } 337 338 static int compare_decl_tags(const void *a, const void *b) 339 { 340 return strverscmp(*(const char **)a, *(const char **)b); 341 } 342 343 /* 344 * Compilers don't guarantee order in which BTF attributes would be generated, 345 * while order is important for test tags like __msg. 346 * Each test tag has the following prefix: "comment:" __COUNTER__, 347 * when sorted using strverscmp this gives same order as in the original C code. 348 */ 349 static const char **collect_decl_tags(struct btf *btf, int id, int *cnt) 350 { 351 const char **tmp, **tags = NULL; 352 const struct btf_type *t; 353 int i; 354 355 *cnt = 0; 356 for (i = 1; i < btf__type_cnt(btf); i++) { 357 t = btf__type_by_id(btf, i); 358 if (!btf_is_decl_tag(t) || t->type != id || btf_decl_tag(t)->component_idx != -1) 359 continue; 360 tmp = realloc(tags, (*cnt + 1) * sizeof(*tags)); 361 if (!tmp) { 362 free(tags); 363 return ERR_PTR(-ENOMEM); 364 } 365 tags = tmp; 366 tags[(*cnt)++] = btf__str_by_offset(btf, t->name_off); 367 } 368 369 if (*cnt) 370 qsort(tags, *cnt, sizeof(*tags), compare_decl_tags); 371 return tags; 372 } 373 374 enum arch { 375 ARCH_UNKNOWN = 0x1, 376 ARCH_X86_64 = 0x2, 377 ARCH_ARM64 = 0x4, 378 ARCH_RISCV64 = 0x8, 379 ARCH_S390X = 0x10, 380 ARCH_LOONGARCH = 0x20, 381 }; 382 383 static int get_current_arch(void) 384 { 385 #if defined(__x86_64__) 386 return ARCH_X86_64; 387 #elif defined(__aarch64__) 388 return ARCH_ARM64; 389 #elif defined(__riscv) && __riscv_xlen == 64 390 return ARCH_RISCV64; 391 #elif defined(__s390x__) 392 return ARCH_S390X; 393 #elif defined(__loongarch__) 394 return ARCH_LOONGARCH; 395 #endif 396 return ARCH_UNKNOWN; 397 } 398 399 /* Uses btf_decl_tag attributes to describe the expected test 400 * behavior, see bpf_misc.h for detailed description of each attribute 401 * and attribute combinations. 402 */ 403 static int parse_test_spec(struct test_loader *tester, 404 struct bpf_object *obj, 405 struct bpf_program *prog, 406 struct test_spec *spec) 407 { 408 const char *description = NULL; 409 bool has_unpriv_result = false; 410 bool has_unpriv_retval = false; 411 bool unpriv_xlated_on_next_line = true; 412 bool xlated_on_next_line = true; 413 bool unpriv_jit_on_next_line; 414 bool jit_on_next_line; 415 bool stderr_on_next_line = true; 416 bool unpriv_stderr_on_next_line = true; 417 bool stdout_on_next_line = true; 418 bool unpriv_stdout_on_next_line = true; 419 bool collect_jit = false; 420 const char **tags = NULL; 421 int func_id, i, nr_tags; 422 int err = 0; 423 u32 arch_mask = 0; 424 u32 load_mask = 0; 425 struct btf *btf; 426 enum arch arch; 427 428 memset(spec, 0, sizeof(*spec)); 429 430 spec->prog_name = bpf_program__name(prog); 431 spec->prog_flags = testing_prog_flags(); 432 433 btf = bpf_object__btf(obj); 434 if (!btf) { 435 ASSERT_FAIL("BPF object has no BTF"); 436 return -EINVAL; 437 } 438 439 func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC); 440 if (func_id < 0) { 441 ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name); 442 return -EINVAL; 443 } 444 445 tags = collect_decl_tags(btf, func_id, &nr_tags); 446 if (IS_ERR(tags)) 447 return PTR_ERR(tags); 448 449 for (i = 0; i < nr_tags; i++) { 450 const char *s, *val, *msg; 451 bool clear; 452 int flags; 453 454 s = skip_decl_tag_pfx(tags[i]); 455 if (!s) 456 continue; 457 if ((val = str_has_pfx(s, "test_description="))) { 458 description = val; 459 } else if (strcmp(s, "test_expect_failure") == 0) { 460 spec->priv.expect_failure = true; 461 spec->mode_mask |= PRIV; 462 } else if (strcmp(s, "test_expect_success") == 0) { 463 spec->priv.expect_failure = false; 464 spec->mode_mask |= PRIV; 465 } else if (strcmp(s, "test_expect_failure_unpriv") == 0) { 466 spec->unpriv.expect_failure = true; 467 spec->mode_mask |= UNPRIV; 468 has_unpriv_result = true; 469 } else if (strcmp(s, "test_expect_success_unpriv") == 0) { 470 spec->unpriv.expect_failure = false; 471 spec->mode_mask |= UNPRIV; 472 has_unpriv_result = true; 473 } else if (strcmp(s, "test_auxiliary") == 0) { 474 spec->auxiliary = true; 475 spec->mode_mask |= PRIV; 476 } else if (strcmp(s, "test_auxiliary_unpriv") == 0) { 477 spec->auxiliary = true; 478 spec->mode_mask |= UNPRIV; 479 } else if ((msg = str_has_pfx(s, "test_expect_msg="))) { 480 err = push_msg(msg, false, &spec->priv.expect_msgs); 481 if (err) 482 goto cleanup; 483 spec->mode_mask |= PRIV; 484 } else if ((msg = str_has_pfx(s, "test_expect_not_msg="))) { 485 err = push_msg(msg, true, &spec->priv.expect_msgs); 486 if (err) 487 goto cleanup; 488 spec->mode_mask |= PRIV; 489 } else if ((msg = str_has_pfx(s, "test_expect_msg_unpriv="))) { 490 err = push_msg(msg, false, &spec->unpriv.expect_msgs); 491 if (err) 492 goto cleanup; 493 spec->mode_mask |= UNPRIV; 494 } else if ((msg = str_has_pfx(s, "test_expect_not_msg_unpriv="))) { 495 err = push_msg(msg, true, &spec->unpriv.expect_msgs); 496 if (err) 497 goto cleanup; 498 spec->mode_mask |= UNPRIV; 499 } else if ((msg = str_has_pfx(s, "test_jited="))) { 500 if (arch_mask == 0) { 501 PRINT_FAIL("__jited used before __arch_*"); 502 goto cleanup; 503 } 504 if (collect_jit) { 505 err = push_disasm_msg(msg, &jit_on_next_line, 506 &spec->priv.jited); 507 if (err) 508 goto cleanup; 509 spec->mode_mask |= PRIV; 510 } 511 } else if ((msg = str_has_pfx(s, "test_jited_unpriv="))) { 512 if (arch_mask == 0) { 513 PRINT_FAIL("__unpriv_jited used before __arch_*"); 514 goto cleanup; 515 } 516 if (collect_jit) { 517 err = push_disasm_msg(msg, &unpriv_jit_on_next_line, 518 &spec->unpriv.jited); 519 if (err) 520 goto cleanup; 521 spec->mode_mask |= UNPRIV; 522 } 523 } else if ((msg = str_has_pfx(s, "test_expect_xlated="))) { 524 err = push_disasm_msg(msg, &xlated_on_next_line, 525 &spec->priv.expect_xlated); 526 if (err) 527 goto cleanup; 528 spec->mode_mask |= PRIV; 529 } else if ((msg = str_has_pfx(s, "test_expect_xlated_unpriv="))) { 530 err = push_disasm_msg(msg, &unpriv_xlated_on_next_line, 531 &spec->unpriv.expect_xlated); 532 if (err) 533 goto cleanup; 534 spec->mode_mask |= UNPRIV; 535 } else if ((val = str_has_pfx(s, "test_retval="))) { 536 err = parse_retval(val, &spec->priv.retval, "__retval"); 537 if (err) 538 goto cleanup; 539 spec->priv.execute = true; 540 spec->mode_mask |= PRIV; 541 } else if ((val = str_has_pfx(s, "test_retval_unpriv="))) { 542 err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv"); 543 if (err) 544 goto cleanup; 545 spec->mode_mask |= UNPRIV; 546 spec->unpriv.execute = true; 547 has_unpriv_retval = true; 548 } else if ((val = str_has_pfx(s, "test_log_level="))) { 549 err = parse_int(val, &spec->log_level, "test log level"); 550 if (err) 551 goto cleanup; 552 } else if ((val = str_has_pfx(s, "test_prog_flags="))) { 553 clear = val[0] == '!'; 554 if (clear) 555 val++; 556 557 if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) { 558 update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear); 559 } else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) { 560 update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear); 561 } else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) { 562 update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear); 563 } else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) { 564 update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear); 565 } else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) { 566 update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear); 567 } else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) { 568 update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); 569 } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { 570 update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); 571 } else /* assume numeric value */ { 572 err = parse_int(val, &flags, "test prog flags"); 573 if (err) 574 goto cleanup; 575 update_flags(&spec->prog_flags, flags, clear); 576 } 577 } else if ((val = str_has_pfx(s, "test_arch="))) { 578 if (strcmp(val, "X86_64") == 0) { 579 arch = ARCH_X86_64; 580 } else if (strcmp(val, "ARM64") == 0) { 581 arch = ARCH_ARM64; 582 } else if (strcmp(val, "RISCV64") == 0) { 583 arch = ARCH_RISCV64; 584 } else if (strcmp(val, "s390x") == 0) { 585 arch = ARCH_S390X; 586 } else if (strcmp(val, "LOONGARCH") == 0) { 587 arch = ARCH_LOONGARCH; 588 } else { 589 PRINT_FAIL("bad arch spec: '%s'\n", val); 590 err = -EINVAL; 591 goto cleanup; 592 } 593 arch_mask |= arch; 594 collect_jit = get_current_arch() == arch; 595 unpriv_jit_on_next_line = true; 596 jit_on_next_line = true; 597 } else if ((val = str_has_pfx(s, "test_btf_path="))) { 598 spec->btf_custom_path = val; 599 } else if ((val = str_has_pfx(s, "test_btf_func_path="))) { 600 spec->btf_custom_func_path = val; 601 } else if ((val = str_has_pfx(s, "test_caps_unpriv="))) { 602 err = parse_caps(val, &spec->unpriv.caps, "test caps"); 603 if (err) 604 goto cleanup; 605 spec->mode_mask |= UNPRIV; 606 } else if ((val = str_has_pfx(s, "load_mode="))) { 607 if (strcmp(val, "jited") == 0) { 608 load_mask = JITED; 609 } else if (strcmp(val, "no_jited") == 0) { 610 load_mask = NO_JITED; 611 } else { 612 PRINT_FAIL("bad load spec: '%s'", val); 613 err = -EINVAL; 614 goto cleanup; 615 } 616 } else if ((msg = str_has_pfx(s, "test_expect_stderr="))) { 617 err = push_disasm_msg(msg, &stderr_on_next_line, 618 &spec->priv.stderr); 619 if (err) 620 goto cleanup; 621 } else if ((msg = str_has_pfx(s, "test_expect_stderr_unpriv="))) { 622 err = push_disasm_msg(msg, &unpriv_stderr_on_next_line, 623 &spec->unpriv.stderr); 624 if (err) 625 goto cleanup; 626 } else if ((msg = str_has_pfx(s, "test_expect_stdout="))) { 627 err = push_disasm_msg(msg, &stdout_on_next_line, 628 &spec->priv.stdout); 629 if (err) 630 goto cleanup; 631 } else if ((msg = str_has_pfx(s, "test_expect_stdout_unpriv="))) { 632 err = push_disasm_msg(msg, &unpriv_stdout_on_next_line, 633 &spec->unpriv.stdout); 634 if (err) 635 goto cleanup; 636 } else if ((val = str_has_pfx(s, "test_linear_size="))) { 637 switch (bpf_program__type(prog)) { 638 case BPF_PROG_TYPE_SCHED_ACT: 639 case BPF_PROG_TYPE_SCHED_CLS: 640 case BPF_PROG_TYPE_CGROUP_SKB: 641 err = parse_int(val, &spec->linear_sz, "test linear size"); 642 if (err) 643 goto cleanup; 644 break; 645 default: 646 PRINT_FAIL("__linear_size for unsupported program type"); 647 err = -EINVAL; 648 goto cleanup; 649 } 650 } 651 } 652 653 spec->arch_mask = arch_mask ?: -1; 654 spec->load_mask = load_mask ?: (JITED | NO_JITED); 655 656 if (spec->mode_mask == 0) 657 spec->mode_mask = PRIV; 658 659 if (spec->mode_mask & PRIV) { 660 spec->priv.name = strdup(spec->prog_name); 661 if (!spec->priv.name) { 662 PRINT_FAIL("failed to allocate memory for priv.name\n"); 663 err = -ENOMEM; 664 goto cleanup; 665 } 666 667 if (description) { 668 spec->priv.description = strdup(description); 669 if (!spec->priv.description) { 670 PRINT_FAIL("failed to allocate memory for priv.description\n"); 671 err = -ENOMEM; 672 goto cleanup; 673 } 674 } 675 } 676 677 if (spec->mode_mask & UNPRIV) { 678 int name_len = strlen(spec->prog_name); 679 const char *suffix = " @unpriv"; 680 int suffix_len = strlen(suffix); 681 char *name; 682 683 name = malloc(name_len + suffix_len + 1); 684 if (!name) { 685 PRINT_FAIL("failed to allocate memory for unpriv.name\n"); 686 err = -ENOMEM; 687 goto cleanup; 688 } 689 690 strcpy(name, spec->prog_name); 691 strcpy(&name[name_len], suffix); 692 spec->unpriv.name = name; 693 694 if (description) { 695 int descr_len = strlen(description); 696 char *descr; 697 698 descr = malloc(descr_len + suffix_len + 1); 699 if (!descr) { 700 PRINT_FAIL("failed to allocate memory for unpriv.description\n"); 701 err = -ENOMEM; 702 goto cleanup; 703 } 704 705 strcpy(descr, description); 706 strcpy(&descr[descr_len], suffix); 707 spec->unpriv.description = descr; 708 } 709 } 710 711 if (spec->mode_mask & (PRIV | UNPRIV)) { 712 if (!has_unpriv_result) 713 spec->unpriv.expect_failure = spec->priv.expect_failure; 714 715 if (!has_unpriv_retval) { 716 spec->unpriv.retval = spec->priv.retval; 717 spec->unpriv.execute = spec->priv.execute; 718 } 719 720 if (spec->unpriv.expect_msgs.cnt == 0) 721 clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs); 722 if (spec->unpriv.expect_xlated.cnt == 0) 723 clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated); 724 if (spec->unpriv.jited.cnt == 0) 725 clone_msgs(&spec->priv.jited, &spec->unpriv.jited); 726 if (spec->unpriv.stderr.cnt == 0) 727 clone_msgs(&spec->priv.stderr, &spec->unpriv.stderr); 728 if (spec->unpriv.stdout.cnt == 0) 729 clone_msgs(&spec->priv.stdout, &spec->unpriv.stdout); 730 } 731 732 spec->valid = true; 733 734 free(tags); 735 return 0; 736 737 cleanup: 738 free(tags); 739 free_test_spec(spec); 740 return err; 741 } 742 743 static void prepare_case(struct test_loader *tester, 744 struct test_spec *spec, 745 struct bpf_object *obj, 746 struct bpf_program *prog) 747 { 748 int min_log_level = 0, prog_flags; 749 750 if (env.verbosity > VERBOSE_NONE) 751 min_log_level = 1; 752 if (env.verbosity > VERBOSE_VERY) 753 min_log_level = 2; 754 755 bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz); 756 757 /* Make sure we set at least minimal log level, unless test requires 758 * even higher level already. Make sure to preserve independent log 759 * level 4 (verifier stats), though. 760 */ 761 if ((spec->log_level & 3) < min_log_level) 762 bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level); 763 else 764 bpf_program__set_log_level(prog, spec->log_level); 765 766 prog_flags = bpf_program__flags(prog); 767 bpf_program__set_flags(prog, prog_flags | spec->prog_flags); 768 769 tester->log_buf[0] = '\0'; 770 } 771 772 static void emit_verifier_log(const char *log_buf, bool force) 773 { 774 if (!force && env.verbosity == VERBOSE_NONE) 775 return; 776 fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf); 777 } 778 779 static void emit_xlated(const char *xlated, bool force) 780 { 781 if (!force && env.verbosity == VERBOSE_NONE) 782 return; 783 fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated); 784 } 785 786 static void emit_jited(const char *jited, bool force) 787 { 788 if (!force && env.verbosity == VERBOSE_NONE) 789 return; 790 fprintf(stdout, "JITED:\n=============\n%s=============\n", jited); 791 } 792 793 static void emit_stderr(const char *stderr, bool force) 794 { 795 if (!force && env.verbosity == VERBOSE_NONE) 796 return; 797 fprintf(stdout, "STDERR:\n=============\n%s=============\n", stderr); 798 } 799 800 static void verify_stderr(int prog_fd, struct expected_msgs *msgs) 801 { 802 LIBBPF_OPTS(bpf_prog_stream_read_opts, ropts); 803 char *buf; 804 int ret; 805 806 if (!msgs->cnt) 807 return; 808 809 buf = malloc(TEST_LOADER_LOG_BUF_SZ); 810 if (!ASSERT_OK_PTR(buf, "malloc")) 811 return; 812 813 ret = bpf_prog_stream_read(prog_fd, 2, buf, TEST_LOADER_LOG_BUF_SZ - 1, 814 &ropts); 815 if (ret > 0) { 816 buf[ret] = '\0'; 817 emit_stderr(buf, false); 818 validate_msgs(buf, msgs, emit_stderr); 819 } else { 820 ASSERT_GT(ret, 0, "stderr stream read"); 821 } 822 823 free(buf); 824 } 825 826 void verify_test_stderr(struct bpf_object *obj, struct bpf_program *prog) 827 { 828 struct test_spec spec = {}; 829 830 if (parse_test_spec(NULL, obj, prog, &spec)) 831 return; 832 833 verify_stderr(bpf_program__fd(prog), &spec.priv.stderr); 834 free_test_spec(&spec); 835 } 836 837 static void emit_stdout(const char *bpf_stdout, bool force) 838 { 839 if (!force && env.verbosity == VERBOSE_NONE) 840 return; 841 fprintf(stdout, "STDOUT:\n=============\n%s=============\n", bpf_stdout); 842 } 843 844 static const char *match_msg(struct expect_msg *msg, const char **log) 845 { 846 const char *match = NULL; 847 regmatch_t reg_match[1]; 848 int err; 849 850 if (!msg->is_regex) { 851 match = strstr(*log, msg->substr); 852 if (match) 853 *log = match + strlen(msg->substr); 854 } else { 855 err = regexec(&msg->regex, *log, 1, reg_match, 0); 856 if (err == 0) { 857 match = *log + reg_match[0].rm_so; 858 *log += reg_match[0].rm_eo; 859 } 860 } 861 return match; 862 } 863 864 static int count_lines(const char *start, const char *end) 865 { 866 const char *tmp; 867 int n = 0; 868 869 for (tmp = start; tmp < end; ++tmp) 870 if (*tmp == '\n') 871 n++; 872 return n; 873 } 874 875 struct match { 876 const char *start; 877 const char *end; 878 int line; 879 }; 880 881 /* 882 * Positive messages are matched sequentially, each next message 883 * is looked for starting from the end of a previous matched one. 884 */ 885 static void match_positive_msgs(const char *log, struct expected_msgs *msgs, struct match *matches) 886 { 887 const char *prev_match; 888 int i, line; 889 890 prev_match = log; 891 line = 0; 892 for (i = 0; i < msgs->cnt; i++) { 893 struct expect_msg *msg = &msgs->patterns[i]; 894 const char *match = NULL; 895 896 if (msg->negative) 897 continue; 898 899 match = match_msg(msg, &log); 900 if (match) { 901 line += count_lines(prev_match, match); 902 matches[i].start = match; 903 matches[i].end = log; 904 matches[i].line = line; 905 prev_match = match; 906 } 907 } 908 } 909 910 /* 911 * Each negative messages N located between positive messages P1 and P2 912 * is matched in the span P1.end .. P2.start. Consequently, negative messages 913 * are unordered within the span. 914 */ 915 static void match_negative_msgs(const char *log, struct expected_msgs *msgs, struct match *matches) 916 { 917 const char *start = log, *end, *next, *match; 918 const char *log_end = log + strlen(log); 919 int i, j, next_positive; 920 921 for (i = 0; i < msgs->cnt; i++) { 922 struct expect_msg *msg = &msgs->patterns[i]; 923 924 /* positive message bumps span start */ 925 if (!msg->negative) { 926 start = matches[i].end ?: start; 927 continue; 928 } 929 930 /* count stride of negative patterns and adjust span end */ 931 end = log_end; 932 for (next_positive = i + 1; next_positive < msgs->cnt; next_positive++) { 933 if (!msgs->patterns[next_positive].negative) { 934 end = matches[next_positive].start; 935 break; 936 } 937 } 938 939 /* try matching negative messages within identified span */ 940 for (j = i; j < next_positive; j++) { 941 next = start; 942 match = match_msg(msg, &next); 943 if (match && next <= end) { 944 matches[j].start = match; 945 matches[j].end = next; 946 } 947 } 948 949 /* -1 to account for i++ */ 950 i = next_positive - 1; 951 } 952 } 953 954 void validate_msgs(const char *log_buf, struct expected_msgs *msgs, 955 void (*emit_fn)(const char *buf, bool force)) 956 { 957 struct match matches[msgs->cnt]; 958 struct match *prev_match = NULL; 959 int i, j; 960 961 memset(matches, 0, sizeof(*matches) * msgs->cnt); 962 match_positive_msgs(log_buf, msgs, matches); 963 match_negative_msgs(log_buf, msgs, matches); 964 965 for (i = 0; i < msgs->cnt; i++) { 966 struct expect_msg *msg = &msgs->patterns[i]; 967 struct match *match = &matches[i]; 968 const char *pat_status; 969 bool unexpected; 970 bool wrong_line; 971 bool no_match; 972 973 no_match = !msg->negative && !match->start; 974 wrong_line = !msg->negative && 975 msg->on_next_line && 976 prev_match && prev_match->line + 1 != match->line; 977 unexpected = msg->negative && match->start; 978 if (no_match || wrong_line || unexpected) { 979 PRINT_FAIL("expect_msg\n"); 980 if (env.verbosity == VERBOSE_NONE) 981 emit_fn(log_buf, true /*force*/); 982 for (j = 0; j <= i; j++) { 983 msg = &msgs->patterns[j]; 984 if (j < i) 985 pat_status = "MATCHED "; 986 else if (wrong_line) 987 pat_status = "WRONG LINE"; 988 else if (no_match) 989 pat_status = "EXPECTED "; 990 else 991 pat_status = "UNEXPECTED"; 992 msg = &msgs->patterns[j]; 993 fprintf(stderr, "%s %s: '%s'\n", 994 pat_status, 995 msg->is_regex ? " REGEX" : "SUBSTR", 996 msg->substr); 997 } 998 if (wrong_line) { 999 fprintf(stderr, 1000 "expecting match at line %d, actual match is at line %d\n", 1001 prev_match->line + 1, match->line); 1002 } 1003 break; 1004 } 1005 1006 if (!msg->negative) 1007 prev_match = match; 1008 } 1009 } 1010 1011 struct cap_state { 1012 __u64 old_caps; 1013 bool initialized; 1014 }; 1015 1016 static int drop_capabilities(struct cap_state *caps) 1017 { 1018 const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN | 1019 1ULL << CAP_PERFMON | 1ULL << CAP_BPF); 1020 int err; 1021 1022 err = cap_disable_effective(caps_to_drop, &caps->old_caps); 1023 if (err) { 1024 PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(-err)); 1025 return err; 1026 } 1027 1028 caps->initialized = true; 1029 return 0; 1030 } 1031 1032 static int restore_capabilities(struct cap_state *caps) 1033 { 1034 int err; 1035 1036 if (!caps->initialized) 1037 return 0; 1038 1039 err = cap_enable_effective(caps->old_caps, NULL); 1040 if (err) 1041 PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(-err)); 1042 caps->initialized = false; 1043 return err; 1044 } 1045 1046 static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec) 1047 { 1048 if (sysctl_unpriv_disabled < 0) 1049 sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0; 1050 if (sysctl_unpriv_disabled) 1051 return false; 1052 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) 1053 return false; 1054 return true; 1055 } 1056 1057 static bool is_unpriv_capable_map(struct bpf_map *map) 1058 { 1059 enum bpf_map_type type; 1060 __u32 flags; 1061 1062 type = bpf_map__type(map); 1063 1064 switch (type) { 1065 case BPF_MAP_TYPE_HASH: 1066 case BPF_MAP_TYPE_PERCPU_HASH: 1067 case BPF_MAP_TYPE_HASH_OF_MAPS: 1068 flags = bpf_map__map_flags(map); 1069 return !(flags & BPF_F_ZERO_SEED); 1070 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 1071 case BPF_MAP_TYPE_ARRAY: 1072 case BPF_MAP_TYPE_RINGBUF: 1073 case BPF_MAP_TYPE_PROG_ARRAY: 1074 case BPF_MAP_TYPE_CGROUP_ARRAY: 1075 case BPF_MAP_TYPE_PERCPU_ARRAY: 1076 case BPF_MAP_TYPE_USER_RINGBUF: 1077 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 1078 case BPF_MAP_TYPE_CGROUP_STORAGE: 1079 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 1080 return true; 1081 default: 1082 return false; 1083 } 1084 } 1085 1086 static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts, int linear_sz) 1087 { 1088 __u8 tmp_out[TEST_DATA_LEN << 2] = {}; 1089 __u8 tmp_in[TEST_DATA_LEN] = {}; 1090 struct __sk_buff ctx = {}; 1091 int err, saved_errno; 1092 LIBBPF_OPTS(bpf_test_run_opts, topts, 1093 .data_in = tmp_in, 1094 .data_size_in = sizeof(tmp_in), 1095 .data_out = tmp_out, 1096 .data_size_out = sizeof(tmp_out), 1097 .repeat = 1, 1098 ); 1099 1100 if (linear_sz) { 1101 ctx.data_end = linear_sz; 1102 topts.ctx_in = &ctx; 1103 topts.ctx_size_in = sizeof(ctx); 1104 } 1105 1106 if (empty_opts) { 1107 memset(&topts, 0, sizeof(struct bpf_test_run_opts)); 1108 topts.sz = sizeof(struct bpf_test_run_opts); 1109 } 1110 err = bpf_prog_test_run_opts(fd_prog, &topts); 1111 saved_errno = errno; 1112 1113 if (err) { 1114 PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ", 1115 saved_errno, strerror(saved_errno)); 1116 return err; 1117 } 1118 1119 ASSERT_OK(0, "bpf_prog_test_run"); 1120 *retval = topts.retval; 1121 1122 return 0; 1123 } 1124 1125 static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec) 1126 { 1127 if (!subspec->execute) 1128 return false; 1129 1130 if (subspec->expect_failure) 1131 return false; 1132 1133 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) { 1134 if (env.verbosity != VERBOSE_NONE) 1135 printf("alignment prevents execution\n"); 1136 return false; 1137 } 1138 1139 return true; 1140 } 1141 1142 /* Get a disassembly of BPF program after verifier applies all rewrites */ 1143 static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz) 1144 { 1145 struct bpf_insn *insn_start = NULL, *insn, *insn_end; 1146 __u32 insns_cnt = 0, i; 1147 char buf[64]; 1148 FILE *out = NULL; 1149 int err; 1150 1151 err = get_xlated_program(prog_fd, &insn_start, &insns_cnt); 1152 if (!ASSERT_OK(err, "get_xlated_program")) 1153 goto out; 1154 out = fmemopen(text, text_sz, "w"); 1155 if (!ASSERT_OK_PTR(out, "open_memstream")) 1156 goto out; 1157 insn_end = insn_start + insns_cnt; 1158 insn = insn_start; 1159 while (insn < insn_end) { 1160 i = insn - insn_start; 1161 insn = disasm_insn(insn, buf, sizeof(buf)); 1162 fprintf(out, "%d: %s\n", i, buf); 1163 } 1164 fflush(out); 1165 1166 out: 1167 free(insn_start); 1168 if (out) 1169 fclose(out); 1170 return err; 1171 } 1172 1173 /* Read the bpf stream corresponding to the stream_id */ 1174 static int get_stream(int stream_id, int prog_fd, char *text, size_t text_sz) 1175 { 1176 LIBBPF_OPTS(bpf_prog_stream_read_opts, ropts); 1177 int ret; 1178 1179 ret = bpf_prog_stream_read(prog_fd, stream_id, text, text_sz, &ropts); 1180 ASSERT_GT(ret, 0, "stream read"); 1181 text[ret] = '\0'; 1182 1183 return ret; 1184 } 1185 1186 /* 1187 * Fix up the program's BTF using BTF from a separate file. 1188 * 1189 * For __naked subprogs, clang drops parameter names from BTF. Find FUNC 1190 * entries with anonymous parameters and replace their FUNC_PROTO with the 1191 * properly-named version from the custom file. 1192 */ 1193 static int fixup_btf_from_path(struct bpf_object *obj, const char *path) 1194 { 1195 struct btf *prog_btf, *custom_btf; 1196 __u32 i, j, cnt, custom_cnt; 1197 int err = 0; 1198 1199 prog_btf = bpf_object__btf(obj); 1200 if (!prog_btf) 1201 return 0; 1202 1203 custom_btf = btf__parse(path, NULL); 1204 if (!ASSERT_OK_PTR(custom_btf, "parse_custom_btf")) 1205 return -EINVAL; 1206 1207 cnt = btf__type_cnt(prog_btf); 1208 custom_cnt = btf__type_cnt(custom_btf); 1209 1210 /* Fix up FUNC entries with anonymous params. 1211 * Save all data from prog_btf BEFORE calling btf__add_*, 1212 * since those calls may reallocate the BTF data buffer 1213 * and invalidate any pointers obtained from btf__type_by_id. 1214 */ 1215 for (i = 1; i < cnt; i++) { 1216 const struct btf_type *t = btf__type_by_id(prog_btf, i); 1217 const struct btf_type *fp, *custom_t, *custom_fp; 1218 const struct btf_param *params, *custom_params; 1219 __u32 ret_type_id, vlen; 1220 __u32 *prog_param_types = NULL; 1221 const char *name; 1222 int new_proto_id; 1223 1224 if (!btf_is_func(t)) 1225 continue; 1226 1227 fp = btf__type_by_id(prog_btf, t->type); 1228 if (!fp || !btf_is_func_proto(fp) || btf_vlen(fp) == 0) 1229 continue; 1230 1231 /* Check if any param is anonymous */ 1232 params = btf_params(fp); 1233 if (params[0].name_off != 0) 1234 continue; 1235 1236 /* Find matching FUNC by name in custom BTF */ 1237 name = btf__name_by_offset(prog_btf, t->name_off); 1238 if (!name) 1239 continue; 1240 1241 for (j = 1; j < custom_cnt; j++) { 1242 const char *cname; 1243 1244 custom_t = btf__type_by_id(custom_btf, j); 1245 if (!btf_is_func(custom_t)) 1246 continue; 1247 cname = btf__name_by_offset(custom_btf, custom_t->name_off); 1248 if (cname && strcmp(name, cname) == 0) 1249 break; 1250 } 1251 if (j >= custom_cnt) 1252 continue; 1253 1254 custom_fp = btf__type_by_id(custom_btf, custom_t->type); 1255 if (!custom_fp || !btf_is_func_proto(custom_fp)) 1256 continue; 1257 1258 vlen = btf_vlen(fp); 1259 if (vlen != btf_vlen(custom_fp)) 1260 continue; 1261 1262 /* Save data before btf__add_* calls invalidate pointers */ 1263 ret_type_id = fp->type; 1264 prog_param_types = malloc(vlen * sizeof(*prog_param_types)); 1265 if (!prog_param_types) { 1266 err = -ENOMEM; 1267 break; 1268 } 1269 for (j = 0; j < vlen; j++) 1270 prog_param_types[j] = params[j].type; 1271 1272 /* Add a new FUNC_PROTO: param names from custom, types from prog */ 1273 new_proto_id = btf__add_func_proto(prog_btf, ret_type_id); 1274 if (new_proto_id < 0) { 1275 err = new_proto_id; 1276 free(prog_param_types); 1277 break; 1278 } 1279 1280 custom_params = btf_params(custom_fp); 1281 for (j = 0; j < vlen; j++) { 1282 const char *pname; 1283 1284 pname = btf__name_by_offset(custom_btf, custom_params[j].name_off); 1285 err = btf__add_func_param(prog_btf, pname ?: "", prog_param_types[j]); 1286 if (err) 1287 break; 1288 } 1289 free(prog_param_types); 1290 if (err) 1291 break; 1292 1293 /* Update the FUNC to point to the new FUNC_PROTO (re-fetch 1294 * since btf__add_* may have reallocated the data buffer). 1295 */ 1296 ((struct btf_type *)btf__type_by_id(prog_btf, i))->type = new_proto_id; 1297 } 1298 1299 btf__free(custom_btf); 1300 return err; 1301 } 1302 1303 /* this function is forced noinline and has short generic name to look better 1304 * in test_progs output (in case of a failure) 1305 */ 1306 static noinline 1307 void run_subtest(struct test_loader *tester, 1308 struct bpf_object_open_opts *open_opts, 1309 const void *obj_bytes, 1310 size_t obj_byte_cnt, 1311 struct test_spec *specs, 1312 struct test_spec *spec, 1313 bool unpriv) 1314 { 1315 struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv; 1316 int current_runtime = is_jit_enabled() ? JITED : NO_JITED; 1317 struct bpf_program *tprog = NULL, *tprog_iter; 1318 struct bpf_link *link, *links[32] = {}; 1319 struct test_spec *spec_iter; 1320 struct cap_state caps = {}; 1321 struct bpf_object *tobj; 1322 struct bpf_map *map; 1323 int retval, err, i; 1324 int links_cnt = 0; 1325 bool should_load; 1326 1327 if (!test__start_subtest_with_desc(subspec->name, subspec->description)) 1328 return; 1329 1330 if ((get_current_arch() & spec->arch_mask) == 0) { 1331 test__skip(); 1332 return; 1333 } 1334 1335 if ((current_runtime & spec->load_mask) == 0) { 1336 test__skip(); 1337 return; 1338 } 1339 1340 if (unpriv) { 1341 if (!can_execute_unpriv(tester, spec)) { 1342 test__skip(); 1343 test__end_subtest(); 1344 return; 1345 } 1346 if (drop_capabilities(&caps)) { 1347 test__end_subtest(); 1348 return; 1349 } 1350 if (subspec->caps) { 1351 err = cap_enable_effective(subspec->caps, NULL); 1352 if (err) { 1353 PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err)); 1354 goto subtest_cleanup; 1355 } 1356 } 1357 } 1358 1359 /* Implicitly reset to NULL if next test case doesn't specify. 1360 * btf_custom_func_path also serves as btf_custom_path for kfunc resolution. 1361 */ 1362 open_opts->btf_custom_path = spec->btf_custom_path; 1363 if (!open_opts->btf_custom_path) 1364 open_opts->btf_custom_path = spec->btf_custom_func_path; 1365 1366 tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts); 1367 if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */ 1368 goto subtest_cleanup; 1369 1370 /* Fix up __naked subprog BTF using a separate file with named params */ 1371 if (spec->btf_custom_func_path) { 1372 err = fixup_btf_from_path(tobj, spec->btf_custom_func_path); 1373 if (err) { 1374 PRINT_FAIL("failed to fixup BTF from %s: %d\n", 1375 spec->btf_custom_func_path, err); 1376 goto tobj_cleanup; 1377 } 1378 } 1379 1380 i = 0; 1381 bpf_object__for_each_program(tprog_iter, tobj) { 1382 spec_iter = &specs[i++]; 1383 should_load = false; 1384 1385 if (spec_iter->valid) { 1386 if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) { 1387 tprog = tprog_iter; 1388 should_load = true; 1389 } 1390 1391 if (spec_iter->auxiliary && 1392 spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV)) 1393 should_load = true; 1394 } 1395 1396 bpf_program__set_autoload(tprog_iter, should_load); 1397 } 1398 1399 prepare_case(tester, spec, tobj, tprog); 1400 1401 /* By default bpf_object__load() automatically creates all 1402 * maps declared in the skeleton. Some map types are only 1403 * allowed in priv mode. Disable autoload for such maps in 1404 * unpriv mode. 1405 */ 1406 bpf_object__for_each_map(map, tobj) 1407 bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map)); 1408 1409 err = bpf_object__load(tobj); 1410 if (subspec->expect_failure) { 1411 if (!ASSERT_ERR(err, "unexpected_load_success")) { 1412 emit_verifier_log(tester->log_buf, false /*force*/); 1413 goto tobj_cleanup; 1414 } 1415 } else { 1416 if (!ASSERT_OK(err, "unexpected_load_failure")) { 1417 emit_verifier_log(tester->log_buf, true /*force*/); 1418 goto tobj_cleanup; 1419 } 1420 } 1421 emit_verifier_log(tester->log_buf, false /*force*/); 1422 validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log); 1423 1424 /* Restore capabilities because the kernel will silently ignore requests 1425 * for program info (such as xlated program text) if we are not 1426 * bpf-capable. Also, for some reason test_verifier executes programs 1427 * with all capabilities restored. Do the same here. 1428 */ 1429 if (restore_capabilities(&caps)) 1430 goto tobj_cleanup; 1431 1432 if (subspec->expect_xlated.cnt) { 1433 err = get_xlated_program_text(bpf_program__fd(tprog), 1434 tester->log_buf, tester->log_buf_sz); 1435 if (err) 1436 goto tobj_cleanup; 1437 emit_xlated(tester->log_buf, false /*force*/); 1438 validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated); 1439 } 1440 1441 if (subspec->jited.cnt) { 1442 err = get_jited_program_text(bpf_program__fd(tprog), 1443 tester->log_buf, tester->log_buf_sz); 1444 if (err == -EOPNOTSUPP) { 1445 printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__); 1446 printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__); 1447 test__skip(); 1448 goto tobj_cleanup; 1449 } 1450 if (!ASSERT_EQ(err, 0, "get_jited_program_text")) 1451 goto tobj_cleanup; 1452 emit_jited(tester->log_buf, false /*force*/); 1453 validate_msgs(tester->log_buf, &subspec->jited, emit_jited); 1454 } 1455 1456 if (should_do_test_run(spec, subspec)) { 1457 /* Do bpf_map__attach_struct_ops() for each struct_ops map. 1458 * This should trigger bpf_struct_ops->reg callback on kernel side. 1459 */ 1460 bpf_object__for_each_map(map, tobj) { 1461 if (!bpf_map__autocreate(map) || 1462 bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS) 1463 continue; 1464 if (links_cnt >= ARRAY_SIZE(links)) { 1465 PRINT_FAIL("too many struct_ops maps"); 1466 goto tobj_cleanup; 1467 } 1468 link = bpf_map__attach_struct_ops(map); 1469 if (!link) { 1470 PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n", 1471 bpf_map__name(map), -errno); 1472 goto tobj_cleanup; 1473 } 1474 links[links_cnt++] = link; 1475 } 1476 1477 if (tester->pre_execution_cb) { 1478 err = tester->pre_execution_cb(tobj); 1479 if (err) { 1480 PRINT_FAIL("pre_execution_cb failed: %d\n", err); 1481 goto tobj_cleanup; 1482 } 1483 } 1484 1485 err = do_prog_test_run(bpf_program__fd(tprog), &retval, 1486 bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false, 1487 spec->linear_sz); 1488 if (!err && retval != subspec->retval && subspec->retval != POINTER_VALUE) { 1489 PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval); 1490 goto tobj_cleanup; 1491 } 1492 1493 verify_stderr(bpf_program__fd(tprog), &subspec->stderr); 1494 1495 if (subspec->stdout.cnt) { 1496 err = get_stream(1, bpf_program__fd(tprog), 1497 tester->log_buf, tester->log_buf_sz); 1498 if (err <= 0) { 1499 PRINT_FAIL("Unexpected retval from get_stream(): %d, errno = %d\n", 1500 err, errno); 1501 goto tobj_cleanup; 1502 } 1503 emit_stdout(tester->log_buf, false /*force*/); 1504 validate_msgs(tester->log_buf, &subspec->stdout, emit_stdout); 1505 } 1506 1507 /* redo bpf_map__attach_struct_ops for each test */ 1508 while (links_cnt > 0) 1509 bpf_link__destroy(links[--links_cnt]); 1510 } 1511 1512 tobj_cleanup: 1513 while (links_cnt > 0) 1514 bpf_link__destroy(links[--links_cnt]); 1515 bpf_object__close(tobj); 1516 subtest_cleanup: 1517 test__end_subtest(); 1518 restore_capabilities(&caps); 1519 } 1520 1521 static void process_subtest(struct test_loader *tester, 1522 const char *skel_name, 1523 skel_elf_bytes_fn elf_bytes_factory) 1524 { 1525 LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name); 1526 struct test_spec *specs = NULL; 1527 struct bpf_object *obj = NULL; 1528 struct bpf_program *prog; 1529 const void *obj_bytes; 1530 int err, i, nr_progs; 1531 size_t obj_byte_cnt; 1532 1533 if (tester_init(tester) < 0) 1534 return; /* failed to initialize tester */ 1535 1536 obj_bytes = elf_bytes_factory(&obj_byte_cnt); 1537 obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts); 1538 if (!ASSERT_OK_PTR(obj, "obj_open_mem")) 1539 return; 1540 1541 nr_progs = 0; 1542 bpf_object__for_each_program(prog, obj) 1543 ++nr_progs; 1544 1545 specs = calloc(nr_progs, sizeof(struct test_spec)); 1546 if (!ASSERT_OK_PTR(specs, "specs_alloc")) 1547 return; 1548 1549 i = 0; 1550 bpf_object__for_each_program(prog, obj) { 1551 /* ignore tests for which we can't derive test specification */ 1552 err = parse_test_spec(tester, obj, prog, &specs[i++]); 1553 if (err) 1554 PRINT_FAIL("Can't parse test spec for program '%s'\n", 1555 bpf_program__name(prog)); 1556 } 1557 1558 i = 0; 1559 bpf_object__for_each_program(prog, obj) { 1560 struct test_spec *spec = &specs[i++]; 1561 1562 if (!spec->valid || spec->auxiliary) 1563 continue; 1564 1565 if (spec->mode_mask & PRIV) 1566 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt, 1567 specs, spec, false); 1568 if (spec->mode_mask & UNPRIV) 1569 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt, 1570 specs, spec, true); 1571 1572 } 1573 1574 for (i = 0; i < nr_progs; ++i) 1575 free_test_spec(&specs[i]); 1576 free(specs); 1577 bpf_object__close(obj); 1578 } 1579 1580 void test_loader__run_subtests(struct test_loader *tester, 1581 const char *skel_name, 1582 skel_elf_bytes_fn elf_bytes_factory) 1583 { 1584 /* see comment in run_subtest() for why we do this function nesting */ 1585 process_subtest(tester, skel_name, elf_bytes_factory); 1586 } 1587