1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/capability.h> 4 #include <linux/err.h> 5 #include <stdlib.h> 6 #include <test_progs.h> 7 #include <bpf/btf.h> 8 9 #include "autoconf_helper.h" 10 #include "disasm_helpers.h" 11 #include "unpriv_helpers.h" 12 #include "cap_helpers.h" 13 #include "jit_disasm_helpers.h" 14 15 static inline const char *str_has_pfx(const char *str, const char *pfx) 16 { 17 size_t len = strlen(pfx); 18 19 return strncmp(str, pfx, len) == 0 ? str + len : NULL; 20 } 21 22 #define TEST_LOADER_LOG_BUF_SZ 2097152 23 24 25 /* Warning: duplicated in bpf_misc.h */ 26 #define POINTER_VALUE 0xbadcafe 27 #define TEST_DATA_LEN 64 28 29 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 30 #define EFFICIENT_UNALIGNED_ACCESS 1 31 #else 32 #define EFFICIENT_UNALIGNED_ACCESS 0 33 #endif 34 35 static int sysctl_unpriv_disabled = -1; 36 static int unpriv_mitigations_disabled = -1; 37 38 enum mode { 39 PRIV = 1, 40 UNPRIV = 2 41 }; 42 43 enum load_mode { 44 JITED = 1 << 0, 45 NO_JITED = 1 << 1, 46 }; 47 48 enum stack_mode { 49 LARGE_STACK = 1 << 0, 50 SMALL_STACK = 1 << 1, 51 }; 52 53 struct test_subspec { 54 char *name; 55 char *description; 56 bool expect_failure; 57 struct expected_msgs expect_msgs; 58 struct expected_msgs expect_xlated; 59 struct expected_msgs jited; 60 struct expected_msgs stderr; 61 struct expected_msgs stdout; 62 int retval; 63 bool execute; 64 __u64 caps; 65 }; 66 67 struct test_spec { 68 const char *prog_name; 69 struct test_subspec priv; 70 struct test_subspec unpriv; 71 const char *btf_custom_path; 72 const char *btf_custom_func_path; 73 int log_level; 74 int prog_flags; 75 int mode_mask; 76 int arch_mask; 77 int load_mask; 78 int stack_mask; 79 int linear_sz; 80 const char *skip_reason; 81 bool prepare_priv; 82 bool auxiliary; 83 bool valid; 84 }; 85 86 static int tester_init(struct test_loader *tester) 87 { 88 if (!tester->log_buf) { 89 tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ; 90 tester->log_buf = calloc(tester->log_buf_sz, 1); 91 if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf")) 92 return -ENOMEM; 93 } 94 95 return 0; 96 } 97 98 void test_loader_fini(struct test_loader *tester) 99 { 100 if (!tester) 101 return; 102 103 free(tester->log_buf); 104 } 105 106 void free_msgs(struct expected_msgs *msgs) 107 { 108 int i; 109 110 for (i = 0; i < msgs->cnt; i++) 111 if (msgs->patterns[i].is_regex) 112 regfree(&msgs->patterns[i].regex); 113 free(msgs->patterns); 114 msgs->patterns = NULL; 115 msgs->cnt = 0; 116 } 117 118 static void free_test_spec(struct test_spec *spec) 119 { 120 /* Deallocate expect_msgs arrays. */ 121 free_msgs(&spec->priv.expect_msgs); 122 free_msgs(&spec->unpriv.expect_msgs); 123 free_msgs(&spec->priv.expect_xlated); 124 free_msgs(&spec->unpriv.expect_xlated); 125 free_msgs(&spec->priv.jited); 126 free_msgs(&spec->unpriv.jited); 127 free_msgs(&spec->unpriv.stderr); 128 free_msgs(&spec->priv.stderr); 129 free_msgs(&spec->unpriv.stdout); 130 free_msgs(&spec->priv.stdout); 131 132 free(spec->priv.name); 133 free(spec->priv.description); 134 free(spec->unpriv.name); 135 free(spec->unpriv.description); 136 spec->priv.name = NULL; 137 spec->priv.description = NULL; 138 spec->unpriv.name = NULL; 139 spec->unpriv.description = NULL; 140 } 141 142 /* Compiles regular expression matching pattern. 143 * Pattern has a special syntax: 144 * 145 * pattern := (<verbatim text> | regex)* 146 * regex := "{{" <posix extended regular expression> "}}" 147 * 148 * In other words, pattern is a verbatim text with inclusion 149 * of regular expressions enclosed in "{{" "}}" pairs. 150 * For example, pattern "foo{{[0-9]+}}" matches strings like 151 * "foo0", "foo007", etc. 152 */ 153 static int compile_regex(const char *pattern, regex_t *regex) 154 { 155 char err_buf[256], buf[256] = {}, *ptr, *buf_end; 156 const char *original_pattern = pattern, *next; 157 bool in_regex = false; 158 int err; 159 160 buf_end = buf + sizeof(buf); 161 ptr = buf; 162 while (*pattern && ptr < buf_end - 2) { 163 if (!in_regex && (next = str_has_pfx(pattern, "{{"))) { 164 in_regex = true; 165 pattern = next; 166 continue; 167 } 168 if (in_regex && (next = str_has_pfx(pattern, "}}"))) { 169 in_regex = false; 170 pattern = next; 171 continue; 172 } 173 if (in_regex) { 174 *ptr++ = *pattern++; 175 continue; 176 } 177 /* list of characters that need escaping for extended posix regex */ 178 if (strchr(".[]\\()*+?{}|^$", *pattern)) { 179 *ptr++ = '\\'; 180 *ptr++ = *pattern++; 181 continue; 182 } 183 *ptr++ = *pattern++; 184 } 185 if (*pattern) { 186 PRINT_FAIL("Regexp too long: '%s'\n", original_pattern); 187 return -EINVAL; 188 } 189 if (in_regex) { 190 PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern); 191 return -EINVAL; 192 } 193 err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE); 194 if (err != 0) { 195 regerror(err, regex, err_buf, sizeof(err_buf)); 196 PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf); 197 return -EINVAL; 198 } 199 return 0; 200 } 201 202 static int __push_msg(const char *pattern, bool on_next_line, bool negative, 203 struct expected_msgs *msgs) 204 { 205 struct expect_msg *msg; 206 void *tmp; 207 int err; 208 209 tmp = realloc(msgs->patterns, 210 (1 + msgs->cnt) * sizeof(struct expect_msg)); 211 if (!tmp) { 212 ASSERT_FAIL("failed to realloc memory for messages\n"); 213 return -ENOMEM; 214 } 215 msgs->patterns = tmp; 216 msg = &msgs->patterns[msgs->cnt]; 217 msg->on_next_line = on_next_line; 218 msg->substr = pattern; 219 msg->negative = negative; 220 msg->is_regex = false; 221 if (strstr(pattern, "{{")) { 222 err = compile_regex(pattern, &msg->regex); 223 if (err) 224 return err; 225 msg->is_regex = true; 226 } 227 msgs->cnt += 1; 228 return 0; 229 } 230 231 static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to) 232 { 233 struct expect_msg *msg; 234 int i, err; 235 236 for (i = 0; i < from->cnt; i++) { 237 msg = &from->patterns[i]; 238 err = __push_msg(msg->substr, msg->on_next_line, msg->negative, to); 239 if (err) 240 return err; 241 } 242 return 0; 243 } 244 245 static int push_msg(const char *substr, bool negative, struct expected_msgs *msgs) 246 { 247 return __push_msg(substr, false, negative, msgs); 248 } 249 250 static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs) 251 { 252 int err; 253 254 if (strcmp(regex_str, "...") == 0) { 255 *on_next_line = false; 256 return 0; 257 } 258 err = __push_msg(regex_str, *on_next_line, false, msgs); 259 if (err) 260 return err; 261 *on_next_line = true; 262 return 0; 263 } 264 265 static int parse_int(const char *str, int *val, const char *name) 266 { 267 char *end; 268 long tmp; 269 270 errno = 0; 271 if (str_has_pfx(str, "0x")) 272 tmp = strtol(str + 2, &end, 16); 273 else 274 tmp = strtol(str, &end, 10); 275 if (errno || end[0] != '\0') { 276 PRINT_FAIL("failed to parse %s from '%s'\n", name, str); 277 return -EINVAL; 278 } 279 *val = tmp; 280 return 0; 281 } 282 283 static int parse_caps(const char *str, __u64 *val, const char *name) 284 { 285 int cap_flag = 0; 286 char *token = NULL, *saveptr = NULL; 287 288 char *str_cpy = strdup(str); 289 if (str_cpy == NULL) { 290 PRINT_FAIL("Memory allocation failed\n"); 291 return -EINVAL; 292 } 293 294 token = strtok_r(str_cpy, "|", &saveptr); 295 while (token != NULL) { 296 errno = 0; 297 if (!strncmp("CAP_", token, sizeof("CAP_") - 1)) { 298 PRINT_FAIL("define %s constant in bpf_misc.h, failed to parse caps\n", token); 299 return -EINVAL; 300 } 301 cap_flag = strtol(token, NULL, 10); 302 if (!cap_flag || errno) { 303 PRINT_FAIL("failed to parse caps %s\n", name); 304 return -EINVAL; 305 } 306 *val |= (1ULL << cap_flag); 307 token = strtok_r(NULL, "|", &saveptr); 308 } 309 310 free(str_cpy); 311 return 0; 312 } 313 314 static int parse_retval(const char *str, int *val, const char *name) 315 { 316 /* 317 * INT_MIN is defined as (-INT_MAX -1), i.e. it doesn't expand to a 318 * single int and cannot be parsed with strtol, so we handle it 319 * separately here. In addition, it expands to different expressions in 320 * different compilers so we use a prefixed _INT_MIN instead. 321 */ 322 if (strcmp(str, "_INT_MIN") == 0) { 323 *val = INT_MIN; 324 return 0; 325 } 326 327 return parse_int(str, val, name); 328 } 329 330 static void update_flags(int *flags, int flag, bool clear) 331 { 332 if (clear) 333 *flags &= ~flag; 334 else 335 *flags |= flag; 336 } 337 338 static const char *skip_decl_tag_pfx(const char *s) 339 { 340 int n = 0; 341 342 if (sscanf(s, "comment:%*d:%n", &n) < 0 || !n) 343 return NULL; 344 return s + n; 345 } 346 347 static int compare_decl_tags(const void *a, const void *b) 348 { 349 return strverscmp(*(const char **)a, *(const char **)b); 350 } 351 352 /* 353 * Compilers don't guarantee order in which BTF attributes would be generated, 354 * while order is important for test tags like __msg. 355 * Each test tag has the following prefix: "comment:" __COUNTER__, 356 * when sorted using strverscmp this gives same order as in the original C code. 357 */ 358 static const char **collect_decl_tags(struct btf *btf, int id, int *cnt) 359 { 360 const char **tmp, **tags = NULL; 361 const struct btf_type *t; 362 int i; 363 364 *cnt = 0; 365 for (i = 1; i < btf__type_cnt(btf); i++) { 366 t = btf__type_by_id(btf, i); 367 if (!btf_is_decl_tag(t) || t->type != id || btf_decl_tag(t)->component_idx != -1) 368 continue; 369 tmp = realloc(tags, (*cnt + 1) * sizeof(*tags)); 370 if (!tmp) { 371 free(tags); 372 return ERR_PTR(-ENOMEM); 373 } 374 tags = tmp; 375 tags[(*cnt)++] = btf__str_by_offset(btf, t->name_off); 376 } 377 378 if (*cnt) 379 qsort(tags, *cnt, sizeof(*tags), compare_decl_tags); 380 return tags; 381 } 382 383 enum arch { 384 ARCH_UNKNOWN = 0x1, 385 ARCH_X86_64 = 0x2, 386 ARCH_ARM64 = 0x4, 387 ARCH_RISCV64 = 0x8, 388 ARCH_S390X = 0x10, 389 ARCH_LOONGARCH = 0x20, 390 }; 391 392 static int get_current_arch(void) 393 { 394 #if defined(__x86_64__) 395 return ARCH_X86_64; 396 #elif defined(__aarch64__) 397 return ARCH_ARM64; 398 #elif defined(__riscv) && __riscv_xlen == 64 399 return ARCH_RISCV64; 400 #elif defined(__s390x__) 401 return ARCH_S390X; 402 #elif defined(__loongarch__) 403 return ARCH_LOONGARCH; 404 #endif 405 return ARCH_UNKNOWN; 406 } 407 408 /* Uses btf_decl_tag attributes to describe the expected test 409 * behavior, see bpf_misc.h for detailed description of each attribute 410 * and attribute combinations. 411 */ 412 static int parse_test_spec(struct test_loader *tester, 413 struct bpf_object *obj, 414 struct bpf_program *prog, 415 struct test_spec *spec) 416 { 417 const char *description = NULL; 418 bool has_unpriv_result = false; 419 bool has_unpriv_retval = false; 420 bool unpriv_xlated_on_next_line = true; 421 bool xlated_on_next_line = true; 422 bool unpriv_jit_on_next_line; 423 bool jit_on_next_line; 424 bool stderr_on_next_line = true; 425 bool unpriv_stderr_on_next_line = true; 426 bool stdout_on_next_line = true; 427 bool unpriv_stdout_on_next_line = true; 428 bool collect_jit = false; 429 const char **tags = NULL; 430 int func_id, i, nr_tags; 431 int err = 0; 432 u32 arch_mask = 0; 433 u32 load_mask = 0; 434 u32 stack_mask = 0; 435 struct btf *btf; 436 enum arch arch; 437 438 memset(spec, 0, sizeof(*spec)); 439 440 spec->prog_name = bpf_program__name(prog); 441 spec->prog_flags = testing_prog_flags(); 442 443 btf = bpf_object__btf(obj); 444 if (!btf) { 445 ASSERT_FAIL("BPF object has no BTF"); 446 return -EINVAL; 447 } 448 449 func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC); 450 if (func_id < 0) { 451 ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name); 452 return -EINVAL; 453 } 454 455 tags = collect_decl_tags(btf, func_id, &nr_tags); 456 if (IS_ERR(tags)) 457 return PTR_ERR(tags); 458 459 for (i = 0; i < nr_tags; i++) { 460 const char *s, *val, *msg; 461 bool clear; 462 int flags; 463 464 s = skip_decl_tag_pfx(tags[i]); 465 if (!s) 466 continue; 467 if ((val = str_has_pfx(s, "test_description="))) { 468 description = val; 469 } else if ((val = str_has_pfx(s, "test_skip="))) { 470 spec->skip_reason = val; 471 } else if (strcmp(s, "test_expect_failure") == 0) { 472 spec->priv.expect_failure = true; 473 spec->mode_mask |= PRIV; 474 } else if (strcmp(s, "test_expect_success") == 0) { 475 spec->priv.expect_failure = false; 476 spec->mode_mask |= PRIV; 477 } else if (strcmp(s, "test_expect_failure_unpriv") == 0) { 478 spec->unpriv.expect_failure = true; 479 spec->mode_mask |= UNPRIV; 480 has_unpriv_result = true; 481 } else if (strcmp(s, "test_expect_success_unpriv") == 0) { 482 spec->unpriv.expect_failure = false; 483 spec->mode_mask |= UNPRIV; 484 has_unpriv_result = true; 485 } else if (strcmp(s, "test_auxiliary") == 0) { 486 spec->auxiliary = true; 487 spec->mode_mask |= PRIV; 488 } else if (strcmp(s, "test_auxiliary_unpriv") == 0) { 489 spec->auxiliary = true; 490 spec->mode_mask |= UNPRIV; 491 } else if ((msg = str_has_pfx(s, "test_expect_msg="))) { 492 err = push_msg(msg, false, &spec->priv.expect_msgs); 493 if (err) 494 goto cleanup; 495 spec->mode_mask |= PRIV; 496 } else if ((msg = str_has_pfx(s, "test_expect_not_msg="))) { 497 err = push_msg(msg, true, &spec->priv.expect_msgs); 498 if (err) 499 goto cleanup; 500 spec->mode_mask |= PRIV; 501 } else if ((msg = str_has_pfx(s, "test_expect_msg_unpriv="))) { 502 err = push_msg(msg, false, &spec->unpriv.expect_msgs); 503 if (err) 504 goto cleanup; 505 spec->mode_mask |= UNPRIV; 506 } else if ((msg = str_has_pfx(s, "test_expect_not_msg_unpriv="))) { 507 err = push_msg(msg, true, &spec->unpriv.expect_msgs); 508 if (err) 509 goto cleanup; 510 spec->mode_mask |= UNPRIV; 511 } else if ((msg = str_has_pfx(s, "test_expect_msg_next="))) { 512 err = __push_msg(msg, true, false, &spec->priv.expect_msgs); 513 if (err) 514 goto cleanup; 515 spec->mode_mask |= PRIV; 516 } else if ((msg = str_has_pfx(s, "test_expect_msg_next_unpriv="))) { 517 err = __push_msg(msg, true, false, &spec->unpriv.expect_msgs); 518 if (err) 519 goto cleanup; 520 spec->mode_mask |= UNPRIV; 521 } else if ((msg = str_has_pfx(s, "test_jited="))) { 522 if (arch_mask == 0) { 523 PRINT_FAIL("__jited used before __arch_*"); 524 goto cleanup; 525 } 526 if (collect_jit) { 527 err = push_disasm_msg(msg, &jit_on_next_line, 528 &spec->priv.jited); 529 if (err) 530 goto cleanup; 531 spec->mode_mask |= PRIV; 532 } 533 } else if ((msg = str_has_pfx(s, "test_jited_unpriv="))) { 534 if (arch_mask == 0) { 535 PRINT_FAIL("__unpriv_jited used before __arch_*"); 536 goto cleanup; 537 } 538 if (collect_jit) { 539 err = push_disasm_msg(msg, &unpriv_jit_on_next_line, 540 &spec->unpriv.jited); 541 if (err) 542 goto cleanup; 543 spec->mode_mask |= UNPRIV; 544 } 545 } else if ((msg = str_has_pfx(s, "test_expect_xlated="))) { 546 err = push_disasm_msg(msg, &xlated_on_next_line, 547 &spec->priv.expect_xlated); 548 if (err) 549 goto cleanup; 550 spec->mode_mask |= PRIV; 551 } else if ((msg = str_has_pfx(s, "test_expect_xlated_unpriv="))) { 552 err = push_disasm_msg(msg, &unpriv_xlated_on_next_line, 553 &spec->unpriv.expect_xlated); 554 if (err) 555 goto cleanup; 556 spec->mode_mask |= UNPRIV; 557 } else if ((val = str_has_pfx(s, "test_retval="))) { 558 err = parse_retval(val, &spec->priv.retval, "__retval"); 559 if (err) 560 goto cleanup; 561 spec->priv.execute = true; 562 spec->mode_mask |= PRIV; 563 } else if ((val = str_has_pfx(s, "test_retval_unpriv="))) { 564 err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv"); 565 if (err) 566 goto cleanup; 567 spec->mode_mask |= UNPRIV; 568 spec->unpriv.execute = true; 569 has_unpriv_retval = true; 570 } else if ((val = str_has_pfx(s, "test_log_level="))) { 571 err = parse_int(val, &spec->log_level, "test log level"); 572 if (err) 573 goto cleanup; 574 } else if ((val = str_has_pfx(s, "test_prog_flags="))) { 575 clear = val[0] == '!'; 576 if (clear) 577 val++; 578 579 if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) { 580 update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear); 581 } else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) { 582 update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear); 583 } else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) { 584 update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear); 585 } else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) { 586 update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear); 587 } else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) { 588 update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear); 589 } else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) { 590 update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); 591 } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { 592 update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); 593 } else if (strcmp(val, "BPF_F_ARENA_SCALAR") == 0) { 594 update_flags(&spec->prog_flags, BPF_F_ARENA_SCALAR, clear); 595 } else /* assume numeric value */ { 596 err = parse_int(val, &flags, "test prog flags"); 597 if (err) 598 goto cleanup; 599 update_flags(&spec->prog_flags, flags, clear); 600 } 601 } else if ((val = str_has_pfx(s, "test_arch="))) { 602 if (strcmp(val, "X86_64") == 0) { 603 arch = ARCH_X86_64; 604 } else if (strcmp(val, "ARM64") == 0) { 605 arch = ARCH_ARM64; 606 } else if (strcmp(val, "RISCV64") == 0) { 607 arch = ARCH_RISCV64; 608 } else if (strcmp(val, "s390x") == 0) { 609 arch = ARCH_S390X; 610 } else if (strcmp(val, "LOONGARCH") == 0) { 611 arch = ARCH_LOONGARCH; 612 } else { 613 PRINT_FAIL("bad arch spec: '%s'\n", val); 614 err = -EINVAL; 615 goto cleanup; 616 } 617 arch_mask |= arch; 618 collect_jit = get_current_arch() == arch; 619 unpriv_jit_on_next_line = true; 620 jit_on_next_line = true; 621 } else if ((val = str_has_pfx(s, "test_btf_path="))) { 622 spec->btf_custom_path = val; 623 } else if ((val = str_has_pfx(s, "test_btf_func_path="))) { 624 spec->btf_custom_func_path = val; 625 } else if ((val = str_has_pfx(s, "test_caps_unpriv="))) { 626 err = parse_caps(val, &spec->unpriv.caps, "test caps"); 627 if (err) 628 goto cleanup; 629 spec->mode_mask |= UNPRIV; 630 } else if (strcmp(s, "test_prepare_priv") == 0) { 631 spec->prepare_priv = true; 632 } else if ((val = str_has_pfx(s, "load_mode="))) { 633 if (strcmp(val, "jited") == 0) { 634 load_mask = JITED; 635 } else if (strcmp(val, "no_jited") == 0) { 636 load_mask = NO_JITED; 637 } else { 638 PRINT_FAIL("bad load spec: '%s'", val); 639 err = -EINVAL; 640 goto cleanup; 641 } 642 } else if ((val = str_has_pfx(s, "stack_mode="))) { 643 if (strcmp(val, "large") == 0) { 644 stack_mask = LARGE_STACK; 645 } else if (strcmp(val, "small") == 0) { 646 stack_mask = SMALL_STACK; 647 } else { 648 PRINT_FAIL("bad stack spec: '%s'", val); 649 err = -EINVAL; 650 goto cleanup; 651 } 652 } else if ((msg = str_has_pfx(s, "test_expect_stderr="))) { 653 err = push_disasm_msg(msg, &stderr_on_next_line, 654 &spec->priv.stderr); 655 if (err) 656 goto cleanup; 657 } else if ((msg = str_has_pfx(s, "test_expect_stderr_unpriv="))) { 658 err = push_disasm_msg(msg, &unpriv_stderr_on_next_line, 659 &spec->unpriv.stderr); 660 if (err) 661 goto cleanup; 662 } else if ((msg = str_has_pfx(s, "test_expect_stdout="))) { 663 err = push_disasm_msg(msg, &stdout_on_next_line, 664 &spec->priv.stdout); 665 if (err) 666 goto cleanup; 667 } else if ((msg = str_has_pfx(s, "test_expect_stdout_unpriv="))) { 668 err = push_disasm_msg(msg, &unpriv_stdout_on_next_line, 669 &spec->unpriv.stdout); 670 if (err) 671 goto cleanup; 672 } else if ((val = str_has_pfx(s, "test_linear_size="))) { 673 switch (bpf_program__type(prog)) { 674 case BPF_PROG_TYPE_SCHED_ACT: 675 case BPF_PROG_TYPE_SCHED_CLS: 676 case BPF_PROG_TYPE_CGROUP_SKB: 677 err = parse_int(val, &spec->linear_sz, "test linear size"); 678 if (err) 679 goto cleanup; 680 break; 681 default: 682 PRINT_FAIL("__linear_size for unsupported program type"); 683 err = -EINVAL; 684 goto cleanup; 685 } 686 } 687 } 688 689 spec->arch_mask = arch_mask ?: -1; 690 spec->load_mask = load_mask ?: (JITED | NO_JITED); 691 spec->stack_mask = stack_mask ?: (LARGE_STACK | SMALL_STACK); 692 693 if (spec->mode_mask == 0) 694 spec->mode_mask = PRIV; 695 696 if (spec->mode_mask & PRIV) { 697 spec->priv.name = strdup(spec->prog_name); 698 if (!spec->priv.name) { 699 PRINT_FAIL("failed to allocate memory for priv.name\n"); 700 err = -ENOMEM; 701 goto cleanup; 702 } 703 704 if (description) { 705 spec->priv.description = strdup(description); 706 if (!spec->priv.description) { 707 PRINT_FAIL("failed to allocate memory for priv.description\n"); 708 err = -ENOMEM; 709 goto cleanup; 710 } 711 } 712 } 713 714 if (spec->mode_mask & UNPRIV) { 715 int name_len = strlen(spec->prog_name); 716 const char *suffix = " @unpriv"; 717 int suffix_len = strlen(suffix); 718 char *name; 719 720 name = malloc(name_len + suffix_len + 1); 721 if (!name) { 722 PRINT_FAIL("failed to allocate memory for unpriv.name\n"); 723 err = -ENOMEM; 724 goto cleanup; 725 } 726 727 strcpy(name, spec->prog_name); 728 strcpy(&name[name_len], suffix); 729 spec->unpriv.name = name; 730 731 if (description) { 732 int descr_len = strlen(description); 733 char *descr; 734 735 descr = malloc(descr_len + suffix_len + 1); 736 if (!descr) { 737 PRINT_FAIL("failed to allocate memory for unpriv.description\n"); 738 err = -ENOMEM; 739 goto cleanup; 740 } 741 742 strcpy(descr, description); 743 strcpy(&descr[descr_len], suffix); 744 spec->unpriv.description = descr; 745 } 746 } 747 748 if (spec->mode_mask & (PRIV | UNPRIV)) { 749 if (!has_unpriv_result) 750 spec->unpriv.expect_failure = spec->priv.expect_failure; 751 752 if (!has_unpriv_retval) { 753 spec->unpriv.retval = spec->priv.retval; 754 spec->unpriv.execute = spec->priv.execute; 755 } 756 757 if (spec->unpriv.expect_msgs.cnt == 0) 758 clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs); 759 if (spec->unpriv.expect_xlated.cnt == 0) 760 clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated); 761 if (spec->unpriv.jited.cnt == 0) 762 clone_msgs(&spec->priv.jited, &spec->unpriv.jited); 763 if (spec->unpriv.stderr.cnt == 0) 764 clone_msgs(&spec->priv.stderr, &spec->unpriv.stderr); 765 if (spec->unpriv.stdout.cnt == 0) 766 clone_msgs(&spec->priv.stdout, &spec->unpriv.stdout); 767 } 768 769 spec->valid = true; 770 771 free(tags); 772 return 0; 773 774 cleanup: 775 free(tags); 776 free_test_spec(spec); 777 return err; 778 } 779 780 static void prepare_case(struct test_loader *tester, 781 struct test_spec *spec, 782 struct bpf_object *obj, 783 struct bpf_program *prog) 784 { 785 int min_log_level = 0; 786 787 if (env.verbosity > VERBOSE_NONE) 788 min_log_level = 1; 789 if (env.verbosity > VERBOSE_VERY) 790 min_log_level = 2; 791 792 bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz); 793 794 /* Make sure we set at least minimal log level, unless test requires 795 * even higher level already. Make sure to preserve independent log 796 * level 4 (verifier stats), though. 797 */ 798 if ((spec->log_level & 3) < min_log_level) 799 bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level); 800 else 801 bpf_program__set_log_level(prog, spec->log_level); 802 803 bpf_program__add_flags(prog, spec->prog_flags); 804 805 tester->log_buf[0] = '\0'; 806 } 807 808 static void emit_verifier_log(const char *log_buf, bool force) 809 { 810 if (!force && env.verbosity == VERBOSE_NONE) 811 return; 812 fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf); 813 } 814 815 static void emit_xlated(const char *xlated, bool force) 816 { 817 if (!force && env.verbosity == VERBOSE_NONE) 818 return; 819 fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated); 820 } 821 822 static void emit_jited(const char *jited, bool force) 823 { 824 if (!force && env.verbosity == VERBOSE_NONE) 825 return; 826 fprintf(stdout, "JITED:\n=============\n%s=============\n", jited); 827 } 828 829 static void emit_stderr(const char *stderr, bool force) 830 { 831 if (!force && env.verbosity == VERBOSE_NONE) 832 return; 833 fprintf(stdout, "STDERR:\n=============\n%s=============\n", stderr); 834 } 835 836 static void verify_stderr(int prog_fd, struct expected_msgs *msgs) 837 { 838 LIBBPF_OPTS(bpf_prog_stream_read_opts, ropts); 839 char *buf; 840 int ret; 841 842 if (!msgs->cnt) 843 return; 844 845 buf = malloc(TEST_LOADER_LOG_BUF_SZ); 846 if (!ASSERT_NEQ(buf, NULL, "malloc")) 847 return; 848 849 ret = bpf_prog_stream_read(prog_fd, 2, buf, TEST_LOADER_LOG_BUF_SZ - 1, 850 &ropts); 851 if (ret > 0) { 852 buf[ret] = '\0'; 853 emit_stderr(buf, false); 854 validate_msgs(buf, msgs, emit_stderr); 855 } else { 856 ASSERT_GT(ret, 0, "stderr stream read"); 857 } 858 859 free(buf); 860 } 861 862 void verify_test_stderr(struct bpf_object *obj, struct bpf_program *prog) 863 { 864 struct test_spec spec = {}; 865 866 if (parse_test_spec(NULL, obj, prog, &spec)) 867 return; 868 869 verify_stderr(bpf_program__fd(prog), &spec.priv.stderr); 870 free_test_spec(&spec); 871 } 872 873 static void emit_stdout(const char *bpf_stdout, bool force) 874 { 875 if (!force && env.verbosity == VERBOSE_NONE) 876 return; 877 fprintf(stdout, "STDOUT:\n=============\n%s=============\n", bpf_stdout); 878 } 879 880 static const char *match_msg(struct expect_msg *msg, const char **log) 881 { 882 const char *match = NULL; 883 regmatch_t reg_match[1]; 884 int err; 885 886 if (!msg->is_regex) { 887 match = strstr(*log, msg->substr); 888 if (match) 889 *log = match + strlen(msg->substr); 890 } else { 891 err = regexec(&msg->regex, *log, 1, reg_match, 0); 892 if (err == 0) { 893 match = *log + reg_match[0].rm_so; 894 *log += reg_match[0].rm_eo; 895 } 896 } 897 return match; 898 } 899 900 static int count_lines(const char *start, const char *end) 901 { 902 const char *tmp; 903 int n = 0; 904 905 for (tmp = start; tmp < end; ++tmp) 906 if (*tmp == '\n') 907 n++; 908 return n; 909 } 910 911 struct match { 912 const char *start; 913 const char *end; 914 int line; 915 }; 916 917 /* 918 * Positive messages are matched sequentially, each next message 919 * is looked for starting from the end of a previous matched one. 920 */ 921 static void match_positive_msgs(const char *log, struct expected_msgs *msgs, struct match *matches) 922 { 923 const char *prev_match; 924 int i, line; 925 926 prev_match = log; 927 line = 0; 928 for (i = 0; i < msgs->cnt; i++) { 929 struct expect_msg *msg = &msgs->patterns[i]; 930 const char *match = NULL; 931 932 if (msg->negative) 933 continue; 934 935 match = match_msg(msg, &log); 936 if (match) { 937 line += count_lines(prev_match, match); 938 matches[i].start = match; 939 matches[i].end = log; 940 matches[i].line = line; 941 prev_match = match; 942 } 943 } 944 } 945 946 /* 947 * Each negative messages N located between positive messages P1 and P2 948 * is matched in the span P1.end .. P2.start. Consequently, negative messages 949 * are unordered within the span. 950 */ 951 static void match_negative_msgs(const char *log, struct expected_msgs *msgs, struct match *matches) 952 { 953 const char *start = log, *end, *next, *match; 954 const char *log_end = log + strlen(log); 955 int i, j, next_positive; 956 957 for (i = 0; i < msgs->cnt; i++) { 958 struct expect_msg *msg = &msgs->patterns[i]; 959 960 /* positive message bumps span start */ 961 if (!msg->negative) { 962 start = matches[i].end ?: start; 963 continue; 964 } 965 966 /* count stride of negative patterns and adjust span end */ 967 end = log_end; 968 for (next_positive = i + 1; next_positive < msgs->cnt; next_positive++) { 969 if (!msgs->patterns[next_positive].negative) { 970 end = matches[next_positive].start; 971 break; 972 } 973 } 974 975 /* try matching negative messages within identified span */ 976 for (j = i; j < next_positive; j++) { 977 next = start; 978 match = match_msg(msg, &next); 979 if (match && next <= end) { 980 matches[j].start = match; 981 matches[j].end = next; 982 } 983 } 984 985 /* -1 to account for i++ */ 986 i = next_positive - 1; 987 } 988 } 989 990 void validate_msgs(const char *log_buf, struct expected_msgs *msgs, 991 void (*emit_fn)(const char *buf, bool force)) 992 { 993 struct match matches[msgs->cnt]; 994 struct match *prev_match = NULL; 995 int i, j; 996 997 memset(matches, 0, sizeof(*matches) * msgs->cnt); 998 match_positive_msgs(log_buf, msgs, matches); 999 match_negative_msgs(log_buf, msgs, matches); 1000 1001 for (i = 0; i < msgs->cnt; i++) { 1002 struct expect_msg *msg = &msgs->patterns[i]; 1003 struct match *match = &matches[i]; 1004 const char *pat_status; 1005 bool unexpected; 1006 bool wrong_line; 1007 bool no_match; 1008 1009 no_match = !msg->negative && !match->start; 1010 wrong_line = !msg->negative && 1011 match->start && 1012 msg->on_next_line && 1013 prev_match && prev_match->line + 1 != match->line; 1014 unexpected = msg->negative && match->start; 1015 if (no_match || wrong_line || unexpected) { 1016 PRINT_FAIL("expect_msg\n"); 1017 if (env.verbosity == VERBOSE_NONE) 1018 emit_fn(log_buf, true /*force*/); 1019 for (j = 0; j <= i; j++) { 1020 msg = &msgs->patterns[j]; 1021 if (j < i) 1022 pat_status = "MATCHED "; 1023 else if (wrong_line) 1024 pat_status = "WRONG LINE"; 1025 else if (no_match) 1026 pat_status = "EXPECTED "; 1027 else 1028 pat_status = "UNEXPECTED"; 1029 msg = &msgs->patterns[j]; 1030 fprintf(stderr, "%s %s: '%s'\n", 1031 pat_status, 1032 msg->is_regex ? " REGEX" : "SUBSTR", 1033 msg->substr); 1034 } 1035 if (wrong_line) { 1036 fprintf(stderr, 1037 "expecting match at line %d, actual match is at line %d\n", 1038 prev_match->line + 1, match->line); 1039 } 1040 break; 1041 } 1042 1043 if (!msg->negative) 1044 prev_match = match; 1045 } 1046 } 1047 1048 struct cap_state { 1049 __u64 old_caps; 1050 bool initialized; 1051 }; 1052 1053 static int drop_capabilities(struct cap_state *caps, __u64 keep_caps) 1054 { 1055 const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN | 1056 1ULL << CAP_PERFMON | 1ULL << CAP_BPF) & ~keep_caps; 1057 int err; 1058 1059 err = cap_disable_effective(caps_to_drop, &caps->old_caps); 1060 if (err) { 1061 PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(-err)); 1062 return err; 1063 } 1064 1065 caps->initialized = true; 1066 if (keep_caps) { 1067 err = cap_enable_effective(keep_caps, NULL); 1068 if (err) { 1069 PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err)); 1070 return err; 1071 } 1072 } 1073 return 0; 1074 } 1075 1076 static int restore_capabilities(struct cap_state *caps) 1077 { 1078 int err; 1079 1080 if (!caps->initialized) 1081 return 0; 1082 1083 err = cap_enable_effective(caps->old_caps, NULL); 1084 if (err) 1085 PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(-err)); 1086 caps->initialized = false; 1087 return err; 1088 } 1089 1090 static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec) 1091 { 1092 if (sysctl_unpriv_disabled < 0) 1093 sysctl_unpriv_disabled = get_unpriv_sysctl_disabled(); 1094 if (sysctl_unpriv_disabled && !(spec->unpriv.caps & (1ULL << CAP_BPF))) 1095 return false; 1096 if (unpriv_mitigations_disabled < 0) 1097 unpriv_mitigations_disabled = get_unpriv_mitigations_disabled(); 1098 if (unpriv_mitigations_disabled) 1099 return false; 1100 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) 1101 return false; 1102 return true; 1103 } 1104 1105 static bool is_unpriv_capable_map(struct bpf_map *map) 1106 { 1107 enum bpf_map_type type; 1108 __u32 flags; 1109 1110 type = bpf_map__type(map); 1111 1112 switch (type) { 1113 case BPF_MAP_TYPE_HASH: 1114 case BPF_MAP_TYPE_PERCPU_HASH: 1115 case BPF_MAP_TYPE_HASH_OF_MAPS: 1116 flags = bpf_map__map_flags(map); 1117 return !(flags & BPF_F_ZERO_SEED); 1118 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 1119 case BPF_MAP_TYPE_ARRAY: 1120 case BPF_MAP_TYPE_RINGBUF: 1121 case BPF_MAP_TYPE_PROG_ARRAY: 1122 case BPF_MAP_TYPE_CGROUP_ARRAY: 1123 case BPF_MAP_TYPE_PERCPU_ARRAY: 1124 case BPF_MAP_TYPE_USER_RINGBUF: 1125 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 1126 case BPF_MAP_TYPE_CGROUP_STORAGE: 1127 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 1128 return true; 1129 default: 1130 return false; 1131 } 1132 } 1133 1134 static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts, int linear_sz) 1135 { 1136 __u8 tmp_out[TEST_DATA_LEN << 2] = {}; 1137 __u8 tmp_in[TEST_DATA_LEN] = {}; 1138 struct __sk_buff ctx = {}; 1139 int err, saved_errno; 1140 LIBBPF_OPTS(bpf_test_run_opts, topts, 1141 .data_in = tmp_in, 1142 .data_size_in = sizeof(tmp_in), 1143 .data_out = tmp_out, 1144 .data_size_out = sizeof(tmp_out), 1145 .repeat = 1, 1146 ); 1147 1148 if (linear_sz) { 1149 ctx.data_end = linear_sz; 1150 topts.ctx_in = &ctx; 1151 topts.ctx_size_in = sizeof(ctx); 1152 } 1153 1154 if (empty_opts) { 1155 memset(&topts, 0, sizeof(struct bpf_test_run_opts)); 1156 topts.sz = sizeof(struct bpf_test_run_opts); 1157 } 1158 err = bpf_prog_test_run_opts(fd_prog, &topts); 1159 saved_errno = errno; 1160 1161 if (err) { 1162 PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ", 1163 saved_errno, strerror(saved_errno)); 1164 return err; 1165 } 1166 1167 ASSERT_OK(0, "bpf_prog_test_run"); 1168 *retval = topts.retval; 1169 1170 return 0; 1171 } 1172 1173 static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec) 1174 { 1175 if (!subspec->execute) 1176 return false; 1177 1178 if (subspec->expect_failure) 1179 return false; 1180 1181 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) { 1182 if (env.verbosity != VERBOSE_NONE) 1183 printf("alignment prevents execution\n"); 1184 return false; 1185 } 1186 1187 return true; 1188 } 1189 1190 /* Get a disassembly of BPF program after verifier applies all rewrites */ 1191 static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz) 1192 { 1193 struct bpf_insn *insn_start = NULL, *insn, *insn_end; 1194 __u32 insns_cnt = 0, i; 1195 char buf[64]; 1196 FILE *out = NULL; 1197 int err; 1198 1199 err = get_xlated_program(prog_fd, &insn_start, &insns_cnt); 1200 if (!ASSERT_OK(err, "get_xlated_program")) 1201 goto out; 1202 out = fmemopen(text, text_sz, "w"); 1203 if (!ASSERT_OK_PTR(out, "open_memstream")) 1204 goto out; 1205 insn_end = insn_start + insns_cnt; 1206 insn = insn_start; 1207 while (insn < insn_end) { 1208 i = insn - insn_start; 1209 insn = disasm_insn(insn, buf, sizeof(buf)); 1210 fprintf(out, "%d: %s\n", i, buf); 1211 } 1212 fflush(out); 1213 1214 out: 1215 free(insn_start); 1216 if (out) 1217 fclose(out); 1218 return err; 1219 } 1220 1221 /* Read the bpf stream corresponding to the stream_id */ 1222 static int get_stream(int stream_id, int prog_fd, char *text, size_t text_sz) 1223 { 1224 LIBBPF_OPTS(bpf_prog_stream_read_opts, ropts); 1225 int ret; 1226 1227 ret = bpf_prog_stream_read(prog_fd, stream_id, text, text_sz, &ropts); 1228 ASSERT_GT(ret, 0, "stream read"); 1229 text[ret] = '\0'; 1230 1231 return ret; 1232 } 1233 1234 /* 1235 * Fix up the program's BTF using BTF from a separate file. 1236 * 1237 * For __naked subprogs, clang drops parameter names from BTF. Find FUNC 1238 * entries with anonymous parameters and replace their FUNC_PROTO with the 1239 * properly-named version from the custom file. 1240 */ 1241 static int fixup_btf_from_path(struct bpf_object *obj, const char *path) 1242 { 1243 struct btf *prog_btf, *custom_btf; 1244 __u32 i, j, cnt, custom_cnt; 1245 int err = 0; 1246 1247 prog_btf = bpf_object__btf(obj); 1248 if (!prog_btf) 1249 return 0; 1250 1251 custom_btf = btf__parse(path, NULL); 1252 if (!ASSERT_OK_PTR(custom_btf, "parse_custom_btf")) 1253 return -EINVAL; 1254 1255 cnt = btf__type_cnt(prog_btf); 1256 custom_cnt = btf__type_cnt(custom_btf); 1257 1258 /* Fix up FUNC entries with anonymous params. 1259 * Save all data from prog_btf BEFORE calling btf__add_*, 1260 * since those calls may reallocate the BTF data buffer 1261 * and invalidate any pointers obtained from btf__type_by_id. 1262 */ 1263 for (i = 1; i < cnt; i++) { 1264 const struct btf_type *t = btf__type_by_id(prog_btf, i); 1265 const struct btf_type *fp, *custom_t, *custom_fp; 1266 const struct btf_param *params, *custom_params; 1267 __u32 ret_type_id, vlen; 1268 __u32 *prog_param_types = NULL; 1269 const char *name; 1270 int new_proto_id; 1271 1272 if (!btf_is_func(t)) 1273 continue; 1274 1275 fp = btf__type_by_id(prog_btf, t->type); 1276 if (!fp || !btf_is_func_proto(fp) || btf_vlen(fp) == 0) 1277 continue; 1278 1279 /* Check if any param is anonymous */ 1280 params = btf_params(fp); 1281 if (params[0].name_off != 0) 1282 continue; 1283 1284 /* Find matching FUNC by name in custom BTF */ 1285 name = btf__name_by_offset(prog_btf, t->name_off); 1286 if (!name) 1287 continue; 1288 1289 for (j = 1; j < custom_cnt; j++) { 1290 const char *cname; 1291 1292 custom_t = btf__type_by_id(custom_btf, j); 1293 if (!btf_is_func(custom_t)) 1294 continue; 1295 cname = btf__name_by_offset(custom_btf, custom_t->name_off); 1296 if (cname && strcmp(name, cname) == 0) 1297 break; 1298 } 1299 if (j >= custom_cnt) 1300 continue; 1301 1302 custom_fp = btf__type_by_id(custom_btf, custom_t->type); 1303 if (!custom_fp || !btf_is_func_proto(custom_fp)) 1304 continue; 1305 1306 vlen = btf_vlen(fp); 1307 if (vlen != btf_vlen(custom_fp)) 1308 continue; 1309 1310 /* Save data before btf__add_* calls invalidate pointers */ 1311 ret_type_id = fp->type; 1312 prog_param_types = malloc(vlen * sizeof(*prog_param_types)); 1313 if (!prog_param_types) { 1314 err = -ENOMEM; 1315 break; 1316 } 1317 for (j = 0; j < vlen; j++) 1318 prog_param_types[j] = params[j].type; 1319 1320 /* Add a new FUNC_PROTO: param names from custom, types from prog */ 1321 new_proto_id = btf__add_func_proto(prog_btf, ret_type_id); 1322 if (new_proto_id < 0) { 1323 err = new_proto_id; 1324 free(prog_param_types); 1325 break; 1326 } 1327 1328 custom_params = btf_params(custom_fp); 1329 for (j = 0; j < vlen; j++) { 1330 const char *pname; 1331 1332 pname = btf__name_by_offset(custom_btf, custom_params[j].name_off); 1333 err = btf__add_func_param(prog_btf, pname ?: "", prog_param_types[j]); 1334 if (err) 1335 break; 1336 } 1337 free(prog_param_types); 1338 if (err) 1339 break; 1340 1341 /* Update the FUNC to point to the new FUNC_PROTO (re-fetch 1342 * since btf__add_* may have reallocated the data buffer). 1343 */ 1344 ((struct btf_type *)btf__type_by_id(prog_btf, i))->type = new_proto_id; 1345 } 1346 1347 btf__free(custom_btf); 1348 return err; 1349 } 1350 1351 /* this function is forced noinline and has short generic name to look better 1352 * in test_progs output (in case of a failure) 1353 */ 1354 static noinline 1355 void run_subtest(struct test_loader *tester, 1356 struct bpf_object_open_opts *open_opts, 1357 const void *obj_bytes, 1358 size_t obj_byte_cnt, 1359 struct test_spec *specs, 1360 struct test_spec *spec, 1361 bool unpriv) 1362 { 1363 struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv; 1364 int current_runtime = is_jit_enabled() ? JITED : NO_JITED; 1365 int current_stack = is_large_stack_supported() ? LARGE_STACK : SMALL_STACK; 1366 struct bpf_program *tprog = NULL, *tprog_iter; 1367 struct bpf_link *link, *links[32] = {}; 1368 struct test_spec *spec_iter; 1369 struct cap_state caps = {}; 1370 struct bpf_object *tobj; 1371 struct bpf_map *map; 1372 int retval, err, i; 1373 int links_cnt = 0; 1374 bool should_load; 1375 1376 if (!test__start_subtest_with_desc(subspec->name, subspec->description)) 1377 return; 1378 1379 if (spec->skip_reason) { 1380 printf("%s:SKIP: %s\n", __func__, spec->skip_reason); 1381 test__skip(); 1382 return; 1383 } 1384 1385 if ((get_current_arch() & spec->arch_mask) == 0) { 1386 test__skip(); 1387 return; 1388 } 1389 1390 if ((current_runtime & spec->load_mask) == 0) { 1391 test__skip(); 1392 return; 1393 } 1394 1395 if ((current_stack & spec->stack_mask) == 0) { 1396 test__skip(); 1397 return; 1398 } 1399 1400 if (unpriv) { 1401 if (!can_execute_unpriv(tester, spec)) { 1402 test__skip(); 1403 test__end_subtest(); 1404 return; 1405 } 1406 if (!spec->prepare_priv && drop_capabilities(&caps, subspec->caps)) 1407 goto subtest_cleanup; 1408 } 1409 1410 /* Implicitly reset to NULL if next test case doesn't specify. 1411 * btf_custom_func_path also serves as btf_custom_path for kfunc resolution. 1412 */ 1413 open_opts->btf_custom_path = spec->btf_custom_path; 1414 if (!open_opts->btf_custom_path) 1415 open_opts->btf_custom_path = spec->btf_custom_func_path; 1416 1417 tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts); 1418 if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */ 1419 goto subtest_cleanup; 1420 1421 /* Fix up __naked subprog BTF using a separate file with named params */ 1422 if (spec->btf_custom_func_path) { 1423 err = fixup_btf_from_path(tobj, spec->btf_custom_func_path); 1424 if (err) { 1425 PRINT_FAIL("failed to fixup BTF from %s: %d\n", 1426 spec->btf_custom_func_path, err); 1427 goto tobj_cleanup; 1428 } 1429 } 1430 1431 i = 0; 1432 bpf_object__for_each_program(tprog_iter, tobj) { 1433 spec_iter = &specs[i++]; 1434 should_load = false; 1435 1436 if (spec_iter->valid) { 1437 if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) { 1438 tprog = tprog_iter; 1439 should_load = true; 1440 } 1441 1442 if (spec_iter->auxiliary && 1443 spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV)) 1444 should_load = true; 1445 } 1446 1447 bpf_program__set_autoload(tprog_iter, should_load); 1448 } 1449 1450 prepare_case(tester, spec, tobj, tprog); 1451 1452 /* By default bpf_object__load() automatically creates all 1453 * maps declared in the skeleton. Some map types are only 1454 * allowed in priv mode. Disable autoload for such maps in 1455 * unpriv mode. 1456 */ 1457 bpf_object__for_each_map(map, tobj) 1458 bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map)); 1459 1460 if (unpriv && spec->prepare_priv) { 1461 /* 1462 * Module BTF lookup needs CAP_SYS_ADMIN. Allow tests to prepare 1463 * their objects first, then verify programs with the requested caps. 1464 */ 1465 err = bpf_object__prepare(tobj); 1466 if (!ASSERT_OK(err, "obj_prepare")) 1467 goto tobj_cleanup; 1468 if (drop_capabilities(&caps, subspec->caps)) 1469 goto tobj_cleanup; 1470 } 1471 1472 err = bpf_object__load(tobj); 1473 if (subspec->expect_failure) { 1474 if (!ASSERT_ERR(err, "unexpected_load_success")) { 1475 emit_verifier_log(tester->log_buf, false /*force*/); 1476 goto tobj_cleanup; 1477 } 1478 } else { 1479 if (!ASSERT_OK(err, "unexpected_load_failure")) { 1480 emit_verifier_log(tester->log_buf, true /*force*/); 1481 goto tobj_cleanup; 1482 } 1483 } 1484 emit_verifier_log(tester->log_buf, false /*force*/); 1485 validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log); 1486 1487 /* Restore capabilities because the kernel will silently ignore requests 1488 * for program info (such as xlated program text) if we are not 1489 * bpf-capable. Also, for some reason test_verifier executes programs 1490 * with all capabilities restored. Do the same here. 1491 */ 1492 if (restore_capabilities(&caps)) 1493 goto tobj_cleanup; 1494 1495 if (subspec->expect_xlated.cnt) { 1496 err = get_xlated_program_text(bpf_program__fd(tprog), 1497 tester->log_buf, tester->log_buf_sz); 1498 if (err) 1499 goto tobj_cleanup; 1500 emit_xlated(tester->log_buf, false /*force*/); 1501 validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated); 1502 } 1503 1504 if (subspec->jited.cnt) { 1505 err = get_jited_program_text(bpf_program__fd(tprog), 1506 tester->log_buf, tester->log_buf_sz); 1507 if (err == -EOPNOTSUPP) { 1508 printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__); 1509 printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__); 1510 test__skip(); 1511 goto tobj_cleanup; 1512 } 1513 if (!ASSERT_EQ(err, 0, "get_jited_program_text")) 1514 goto tobj_cleanup; 1515 emit_jited(tester->log_buf, false /*force*/); 1516 validate_msgs(tester->log_buf, &subspec->jited, emit_jited); 1517 } 1518 1519 if (should_do_test_run(spec, subspec)) { 1520 /* Do bpf_map__attach_struct_ops() for each struct_ops map. 1521 * This should trigger bpf_struct_ops->reg callback on kernel side. 1522 */ 1523 bpf_object__for_each_map(map, tobj) { 1524 if (!bpf_map__autocreate(map) || 1525 bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS) 1526 continue; 1527 if (links_cnt >= ARRAY_SIZE(links)) { 1528 PRINT_FAIL("too many struct_ops maps"); 1529 goto tobj_cleanup; 1530 } 1531 link = bpf_map__attach_struct_ops(map); 1532 if (!link) { 1533 PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n", 1534 bpf_map__name(map), -errno); 1535 goto tobj_cleanup; 1536 } 1537 links[links_cnt++] = link; 1538 } 1539 1540 if (tester->pre_execution_cb) { 1541 err = tester->pre_execution_cb(tobj); 1542 if (err) { 1543 PRINT_FAIL("pre_execution_cb failed: %d\n", err); 1544 goto tobj_cleanup; 1545 } 1546 } 1547 1548 err = do_prog_test_run(bpf_program__fd(tprog), &retval, 1549 bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false, 1550 spec->linear_sz); 1551 if (!err && retval != subspec->retval && subspec->retval != POINTER_VALUE) { 1552 PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval); 1553 goto tobj_cleanup; 1554 } 1555 1556 verify_stderr(bpf_program__fd(tprog), &subspec->stderr); 1557 1558 if (subspec->stdout.cnt) { 1559 err = get_stream(1, bpf_program__fd(tprog), 1560 tester->log_buf, tester->log_buf_sz); 1561 if (err <= 0) { 1562 PRINT_FAIL("Unexpected retval from get_stream(): %d, errno = %d\n", 1563 err, errno); 1564 goto tobj_cleanup; 1565 } 1566 emit_stdout(tester->log_buf, false /*force*/); 1567 validate_msgs(tester->log_buf, &subspec->stdout, emit_stdout); 1568 } 1569 1570 /* redo bpf_map__attach_struct_ops for each test */ 1571 while (links_cnt > 0) 1572 bpf_link__destroy(links[--links_cnt]); 1573 } 1574 1575 tobj_cleanup: 1576 while (links_cnt > 0) 1577 bpf_link__destroy(links[--links_cnt]); 1578 bpf_object__close(tobj); 1579 subtest_cleanup: 1580 test__end_subtest(); 1581 restore_capabilities(&caps); 1582 } 1583 1584 static void process_subtest(struct test_loader *tester, 1585 const char *skel_name, 1586 skel_elf_bytes_fn elf_bytes_factory) 1587 { 1588 LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name); 1589 struct test_spec *specs = NULL; 1590 struct bpf_object *obj = NULL; 1591 struct bpf_program *prog; 1592 const void *obj_bytes; 1593 int err, i, nr_progs; 1594 size_t obj_byte_cnt; 1595 1596 if (tester_init(tester) < 0) 1597 return; /* failed to initialize tester */ 1598 1599 obj_bytes = elf_bytes_factory(&obj_byte_cnt); 1600 obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts); 1601 if (!ASSERT_OK_PTR(obj, "obj_open_mem")) 1602 return; 1603 1604 nr_progs = 0; 1605 bpf_object__for_each_program(prog, obj) 1606 ++nr_progs; 1607 1608 specs = calloc(nr_progs, sizeof(struct test_spec)); 1609 if (!ASSERT_OK_PTR(specs, "specs_alloc")) 1610 return; 1611 1612 i = 0; 1613 bpf_object__for_each_program(prog, obj) { 1614 /* ignore tests for which we can't derive test specification */ 1615 err = parse_test_spec(tester, obj, prog, &specs[i++]); 1616 if (err) 1617 PRINT_FAIL("Can't parse test spec for program '%s'\n", 1618 bpf_program__name(prog)); 1619 } 1620 1621 i = 0; 1622 bpf_object__for_each_program(prog, obj) { 1623 struct test_spec *spec = &specs[i++]; 1624 1625 if (!spec->valid || spec->auxiliary) 1626 continue; 1627 1628 if (spec->mode_mask & PRIV) 1629 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt, 1630 specs, spec, false); 1631 if (spec->mode_mask & UNPRIV) 1632 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt, 1633 specs, spec, true); 1634 1635 } 1636 1637 for (i = 0; i < nr_progs; ++i) 1638 free_test_spec(&specs[i]); 1639 free(specs); 1640 bpf_object__close(obj); 1641 } 1642 1643 void test_loader__run_subtests(struct test_loader *tester, 1644 const char *skel_name, 1645 skel_elf_bytes_fn elf_bytes_factory) 1646 { 1647 /* see comment in run_subtest() for why we do this function nesting */ 1648 process_subtest(tester, skel_name, elf_bytes_factory); 1649 } 1650