1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/capability.h> 4 #include <stdlib.h> 5 #include <regex.h> 6 #include <test_progs.h> 7 #include <bpf/btf.h> 8 9 #include "autoconf_helper.h" 10 #include "disasm_helpers.h" 11 #include "unpriv_helpers.h" 12 #include "cap_helpers.h" 13 #include "jit_disasm_helpers.h" 14 15 #define str_has_pfx(str, pfx) \ 16 (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0) 17 18 #define TEST_LOADER_LOG_BUF_SZ 2097152 19 20 #define TEST_TAG_EXPECT_FAILURE "comment:test_expect_failure" 21 #define TEST_TAG_EXPECT_SUCCESS "comment:test_expect_success" 22 #define TEST_TAG_EXPECT_MSG_PFX "comment:test_expect_msg=" 23 #define TEST_TAG_EXPECT_XLATED_PFX "comment:test_expect_xlated=" 24 #define TEST_TAG_EXPECT_FAILURE_UNPRIV "comment:test_expect_failure_unpriv" 25 #define TEST_TAG_EXPECT_SUCCESS_UNPRIV "comment:test_expect_success_unpriv" 26 #define TEST_TAG_EXPECT_MSG_PFX_UNPRIV "comment:test_expect_msg_unpriv=" 27 #define TEST_TAG_EXPECT_XLATED_PFX_UNPRIV "comment:test_expect_xlated_unpriv=" 28 #define TEST_TAG_LOG_LEVEL_PFX "comment:test_log_level=" 29 #define TEST_TAG_PROG_FLAGS_PFX "comment:test_prog_flags=" 30 #define TEST_TAG_DESCRIPTION_PFX "comment:test_description=" 31 #define TEST_TAG_RETVAL_PFX "comment:test_retval=" 32 #define TEST_TAG_RETVAL_PFX_UNPRIV "comment:test_retval_unpriv=" 33 #define TEST_TAG_AUXILIARY "comment:test_auxiliary" 34 #define TEST_TAG_AUXILIARY_UNPRIV "comment:test_auxiliary_unpriv" 35 #define TEST_BTF_PATH "comment:test_btf_path=" 36 #define TEST_TAG_ARCH "comment:test_arch=" 37 #define TEST_TAG_JITED_PFX "comment:test_jited=" 38 #define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv=" 39 40 /* Warning: duplicated in bpf_misc.h */ 41 #define POINTER_VALUE 0xcafe4all 42 #define TEST_DATA_LEN 64 43 44 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 45 #define EFFICIENT_UNALIGNED_ACCESS 1 46 #else 47 #define EFFICIENT_UNALIGNED_ACCESS 0 48 #endif 49 50 static int sysctl_unpriv_disabled = -1; 51 52 enum mode { 53 PRIV = 1, 54 UNPRIV = 2 55 }; 56 57 struct expect_msg { 58 const char *substr; /* substring match */ 59 regex_t regex; 60 bool is_regex; 61 bool on_next_line; 62 }; 63 64 struct expected_msgs { 65 struct expect_msg *patterns; 66 size_t cnt; 67 }; 68 69 struct test_subspec { 70 char *name; 71 bool expect_failure; 72 struct expected_msgs expect_msgs; 73 struct expected_msgs expect_xlated; 74 struct expected_msgs jited; 75 int retval; 76 bool execute; 77 }; 78 79 struct test_spec { 80 const char *prog_name; 81 struct test_subspec priv; 82 struct test_subspec unpriv; 83 const char *btf_custom_path; 84 int log_level; 85 int prog_flags; 86 int mode_mask; 87 int arch_mask; 88 bool auxiliary; 89 bool valid; 90 }; 91 92 static int tester_init(struct test_loader *tester) 93 { 94 if (!tester->log_buf) { 95 tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ; 96 tester->log_buf = calloc(tester->log_buf_sz, 1); 97 if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf")) 98 return -ENOMEM; 99 } 100 101 return 0; 102 } 103 104 void test_loader_fini(struct test_loader *tester) 105 { 106 if (!tester) 107 return; 108 109 free(tester->log_buf); 110 } 111 112 static void free_msgs(struct expected_msgs *msgs) 113 { 114 int i; 115 116 for (i = 0; i < msgs->cnt; i++) 117 if (msgs->patterns[i].is_regex) 118 regfree(&msgs->patterns[i].regex); 119 free(msgs->patterns); 120 msgs->patterns = NULL; 121 msgs->cnt = 0; 122 } 123 124 static void free_test_spec(struct test_spec *spec) 125 { 126 /* Deallocate expect_msgs arrays. */ 127 free_msgs(&spec->priv.expect_msgs); 128 free_msgs(&spec->unpriv.expect_msgs); 129 free_msgs(&spec->priv.expect_xlated); 130 free_msgs(&spec->unpriv.expect_xlated); 131 free_msgs(&spec->priv.jited); 132 free_msgs(&spec->unpriv.jited); 133 134 free(spec->priv.name); 135 free(spec->unpriv.name); 136 spec->priv.name = NULL; 137 spec->unpriv.name = NULL; 138 } 139 140 /* Compiles regular expression matching pattern. 141 * Pattern has a special syntax: 142 * 143 * pattern := (<verbatim text> | regex)* 144 * regex := "{{" <posix extended regular expression> "}}" 145 * 146 * In other words, pattern is a verbatim text with inclusion 147 * of regular expressions enclosed in "{{" "}}" pairs. 148 * For example, pattern "foo{{[0-9]+}}" matches strings like 149 * "foo0", "foo007", etc. 150 */ 151 static int compile_regex(const char *pattern, regex_t *regex) 152 { 153 char err_buf[256], buf[256] = {}, *ptr, *buf_end; 154 const char *original_pattern = pattern; 155 bool in_regex = false; 156 int err; 157 158 buf_end = buf + sizeof(buf); 159 ptr = buf; 160 while (*pattern && ptr < buf_end - 2) { 161 if (!in_regex && str_has_pfx(pattern, "{{")) { 162 in_regex = true; 163 pattern += 2; 164 continue; 165 } 166 if (in_regex && str_has_pfx(pattern, "}}")) { 167 in_regex = false; 168 pattern += 2; 169 continue; 170 } 171 if (in_regex) { 172 *ptr++ = *pattern++; 173 continue; 174 } 175 /* list of characters that need escaping for extended posix regex */ 176 if (strchr(".[]\\()*+?{}|^$", *pattern)) { 177 *ptr++ = '\\'; 178 *ptr++ = *pattern++; 179 continue; 180 } 181 *ptr++ = *pattern++; 182 } 183 if (*pattern) { 184 PRINT_FAIL("Regexp too long: '%s'\n", original_pattern); 185 return -EINVAL; 186 } 187 if (in_regex) { 188 PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern); 189 return -EINVAL; 190 } 191 err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE); 192 if (err != 0) { 193 regerror(err, regex, err_buf, sizeof(err_buf)); 194 PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf); 195 return -EINVAL; 196 } 197 return 0; 198 } 199 200 static int __push_msg(const char *pattern, bool on_next_line, struct expected_msgs *msgs) 201 { 202 struct expect_msg *msg; 203 void *tmp; 204 int err; 205 206 tmp = realloc(msgs->patterns, 207 (1 + msgs->cnt) * sizeof(struct expect_msg)); 208 if (!tmp) { 209 ASSERT_FAIL("failed to realloc memory for messages\n"); 210 return -ENOMEM; 211 } 212 msgs->patterns = tmp; 213 msg = &msgs->patterns[msgs->cnt]; 214 msg->on_next_line = on_next_line; 215 msg->substr = pattern; 216 msg->is_regex = false; 217 if (strstr(pattern, "{{")) { 218 err = compile_regex(pattern, &msg->regex); 219 if (err) 220 return err; 221 msg->is_regex = true; 222 } 223 msgs->cnt += 1; 224 return 0; 225 } 226 227 static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to) 228 { 229 struct expect_msg *msg; 230 int i, err; 231 232 for (i = 0; i < from->cnt; i++) { 233 msg = &from->patterns[i]; 234 err = __push_msg(msg->substr, msg->on_next_line, to); 235 if (err) 236 return err; 237 } 238 return 0; 239 } 240 241 static int push_msg(const char *substr, struct expected_msgs *msgs) 242 { 243 return __push_msg(substr, false, msgs); 244 } 245 246 static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs) 247 { 248 int err; 249 250 if (strcmp(regex_str, "...") == 0) { 251 *on_next_line = false; 252 return 0; 253 } 254 err = __push_msg(regex_str, *on_next_line, msgs); 255 if (err) 256 return err; 257 *on_next_line = true; 258 return 0; 259 } 260 261 static int parse_int(const char *str, int *val, const char *name) 262 { 263 char *end; 264 long tmp; 265 266 errno = 0; 267 if (str_has_pfx(str, "0x")) 268 tmp = strtol(str + 2, &end, 16); 269 else 270 tmp = strtol(str, &end, 10); 271 if (errno || end[0] != '\0') { 272 PRINT_FAIL("failed to parse %s from '%s'\n", name, str); 273 return -EINVAL; 274 } 275 *val = tmp; 276 return 0; 277 } 278 279 static int parse_retval(const char *str, int *val, const char *name) 280 { 281 struct { 282 char *name; 283 int val; 284 } named_values[] = { 285 { "INT_MIN" , INT_MIN }, 286 { "POINTER_VALUE", POINTER_VALUE }, 287 { "TEST_DATA_LEN", TEST_DATA_LEN }, 288 }; 289 int i; 290 291 for (i = 0; i < ARRAY_SIZE(named_values); ++i) { 292 if (strcmp(str, named_values[i].name) != 0) 293 continue; 294 *val = named_values[i].val; 295 return 0; 296 } 297 298 return parse_int(str, val, name); 299 } 300 301 static void update_flags(int *flags, int flag, bool clear) 302 { 303 if (clear) 304 *flags &= ~flag; 305 else 306 *flags |= flag; 307 } 308 309 /* Matches a string of form '<pfx>[^=]=.*' and returns it's suffix. 310 * Used to parse btf_decl_tag values. 311 * Such values require unique prefix because compiler does not add 312 * same __attribute__((btf_decl_tag(...))) twice. 313 * Test suite uses two-component tags for such cases: 314 * 315 * <pfx> __COUNTER__ '=' 316 * 317 * For example, two consecutive __msg tags '__msg("foo") __msg("foo")' 318 * would be encoded as: 319 * 320 * [18] DECL_TAG 'comment:test_expect_msg=0=foo' type_id=15 component_idx=-1 321 * [19] DECL_TAG 'comment:test_expect_msg=1=foo' type_id=15 component_idx=-1 322 * 323 * And the purpose of this function is to extract 'foo' from the above. 324 */ 325 static const char *skip_dynamic_pfx(const char *s, const char *pfx) 326 { 327 const char *msg; 328 329 if (strncmp(s, pfx, strlen(pfx)) != 0) 330 return NULL; 331 msg = s + strlen(pfx); 332 msg = strchr(msg, '='); 333 if (!msg) 334 return NULL; 335 return msg + 1; 336 } 337 338 enum arch { 339 ARCH_UNKNOWN = 0x1, 340 ARCH_X86_64 = 0x2, 341 ARCH_ARM64 = 0x4, 342 ARCH_RISCV64 = 0x8, 343 }; 344 345 static int get_current_arch(void) 346 { 347 #if defined(__x86_64__) 348 return ARCH_X86_64; 349 #elif defined(__aarch64__) 350 return ARCH_ARM64; 351 #elif defined(__riscv) && __riscv_xlen == 64 352 return ARCH_RISCV64; 353 #endif 354 return ARCH_UNKNOWN; 355 } 356 357 /* Uses btf_decl_tag attributes to describe the expected test 358 * behavior, see bpf_misc.h for detailed description of each attribute 359 * and attribute combinations. 360 */ 361 static int parse_test_spec(struct test_loader *tester, 362 struct bpf_object *obj, 363 struct bpf_program *prog, 364 struct test_spec *spec) 365 { 366 const char *description = NULL; 367 bool has_unpriv_result = false; 368 bool has_unpriv_retval = false; 369 bool unpriv_xlated_on_next_line = true; 370 bool xlated_on_next_line = true; 371 bool unpriv_jit_on_next_line; 372 bool jit_on_next_line; 373 bool collect_jit = false; 374 int func_id, i, err = 0; 375 u32 arch_mask = 0; 376 struct btf *btf; 377 enum arch arch; 378 379 memset(spec, 0, sizeof(*spec)); 380 381 spec->prog_name = bpf_program__name(prog); 382 spec->prog_flags = testing_prog_flags(); 383 384 btf = bpf_object__btf(obj); 385 if (!btf) { 386 ASSERT_FAIL("BPF object has no BTF"); 387 return -EINVAL; 388 } 389 390 func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC); 391 if (func_id < 0) { 392 ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name); 393 return -EINVAL; 394 } 395 396 for (i = 1; i < btf__type_cnt(btf); i++) { 397 const char *s, *val, *msg; 398 const struct btf_type *t; 399 bool clear; 400 int flags; 401 402 t = btf__type_by_id(btf, i); 403 if (!btf_is_decl_tag(t)) 404 continue; 405 406 if (t->type != func_id || btf_decl_tag(t)->component_idx != -1) 407 continue; 408 409 s = btf__str_by_offset(btf, t->name_off); 410 if (str_has_pfx(s, TEST_TAG_DESCRIPTION_PFX)) { 411 description = s + sizeof(TEST_TAG_DESCRIPTION_PFX) - 1; 412 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE) == 0) { 413 spec->priv.expect_failure = true; 414 spec->mode_mask |= PRIV; 415 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS) == 0) { 416 spec->priv.expect_failure = false; 417 spec->mode_mask |= PRIV; 418 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE_UNPRIV) == 0) { 419 spec->unpriv.expect_failure = true; 420 spec->mode_mask |= UNPRIV; 421 has_unpriv_result = true; 422 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS_UNPRIV) == 0) { 423 spec->unpriv.expect_failure = false; 424 spec->mode_mask |= UNPRIV; 425 has_unpriv_result = true; 426 } else if (strcmp(s, TEST_TAG_AUXILIARY) == 0) { 427 spec->auxiliary = true; 428 spec->mode_mask |= PRIV; 429 } else if (strcmp(s, TEST_TAG_AUXILIARY_UNPRIV) == 0) { 430 spec->auxiliary = true; 431 spec->mode_mask |= UNPRIV; 432 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX))) { 433 err = push_msg(msg, &spec->priv.expect_msgs); 434 if (err) 435 goto cleanup; 436 spec->mode_mask |= PRIV; 437 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX_UNPRIV))) { 438 err = push_msg(msg, &spec->unpriv.expect_msgs); 439 if (err) 440 goto cleanup; 441 spec->mode_mask |= UNPRIV; 442 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX))) { 443 if (arch_mask == 0) { 444 PRINT_FAIL("__jited used before __arch_*"); 445 goto cleanup; 446 } 447 if (collect_jit) { 448 err = push_disasm_msg(msg, &jit_on_next_line, 449 &spec->priv.jited); 450 if (err) 451 goto cleanup; 452 spec->mode_mask |= PRIV; 453 } 454 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX_UNPRIV))) { 455 if (arch_mask == 0) { 456 PRINT_FAIL("__unpriv_jited used before __arch_*"); 457 goto cleanup; 458 } 459 if (collect_jit) { 460 err = push_disasm_msg(msg, &unpriv_jit_on_next_line, 461 &spec->unpriv.jited); 462 if (err) 463 goto cleanup; 464 spec->mode_mask |= UNPRIV; 465 } 466 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX))) { 467 err = push_disasm_msg(msg, &xlated_on_next_line, 468 &spec->priv.expect_xlated); 469 if (err) 470 goto cleanup; 471 spec->mode_mask |= PRIV; 472 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX_UNPRIV))) { 473 err = push_disasm_msg(msg, &unpriv_xlated_on_next_line, 474 &spec->unpriv.expect_xlated); 475 if (err) 476 goto cleanup; 477 spec->mode_mask |= UNPRIV; 478 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX)) { 479 val = s + sizeof(TEST_TAG_RETVAL_PFX) - 1; 480 err = parse_retval(val, &spec->priv.retval, "__retval"); 481 if (err) 482 goto cleanup; 483 spec->priv.execute = true; 484 spec->mode_mask |= PRIV; 485 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX_UNPRIV)) { 486 val = s + sizeof(TEST_TAG_RETVAL_PFX_UNPRIV) - 1; 487 err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv"); 488 if (err) 489 goto cleanup; 490 spec->mode_mask |= UNPRIV; 491 spec->unpriv.execute = true; 492 has_unpriv_retval = true; 493 } else if (str_has_pfx(s, TEST_TAG_LOG_LEVEL_PFX)) { 494 val = s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1; 495 err = parse_int(val, &spec->log_level, "test log level"); 496 if (err) 497 goto cleanup; 498 } else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) { 499 val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1; 500 501 clear = val[0] == '!'; 502 if (clear) 503 val++; 504 505 if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) { 506 update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear); 507 } else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) { 508 update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear); 509 } else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) { 510 update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear); 511 } else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) { 512 update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear); 513 } else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) { 514 update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear); 515 } else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) { 516 update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); 517 } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { 518 update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); 519 } else /* assume numeric value */ { 520 err = parse_int(val, &flags, "test prog flags"); 521 if (err) 522 goto cleanup; 523 update_flags(&spec->prog_flags, flags, clear); 524 } 525 } else if (str_has_pfx(s, TEST_TAG_ARCH)) { 526 val = s + sizeof(TEST_TAG_ARCH) - 1; 527 if (strcmp(val, "X86_64") == 0) { 528 arch = ARCH_X86_64; 529 } else if (strcmp(val, "ARM64") == 0) { 530 arch = ARCH_ARM64; 531 } else if (strcmp(val, "RISCV64") == 0) { 532 arch = ARCH_RISCV64; 533 } else { 534 PRINT_FAIL("bad arch spec: '%s'", val); 535 err = -EINVAL; 536 goto cleanup; 537 } 538 arch_mask |= arch; 539 collect_jit = get_current_arch() == arch; 540 unpriv_jit_on_next_line = true; 541 jit_on_next_line = true; 542 } else if (str_has_pfx(s, TEST_BTF_PATH)) { 543 spec->btf_custom_path = s + sizeof(TEST_BTF_PATH) - 1; 544 } 545 } 546 547 spec->arch_mask = arch_mask ?: -1; 548 549 if (spec->mode_mask == 0) 550 spec->mode_mask = PRIV; 551 552 if (!description) 553 description = spec->prog_name; 554 555 if (spec->mode_mask & PRIV) { 556 spec->priv.name = strdup(description); 557 if (!spec->priv.name) { 558 PRINT_FAIL("failed to allocate memory for priv.name\n"); 559 err = -ENOMEM; 560 goto cleanup; 561 } 562 } 563 564 if (spec->mode_mask & UNPRIV) { 565 int descr_len = strlen(description); 566 const char *suffix = " @unpriv"; 567 char *name; 568 569 name = malloc(descr_len + strlen(suffix) + 1); 570 if (!name) { 571 PRINT_FAIL("failed to allocate memory for unpriv.name\n"); 572 err = -ENOMEM; 573 goto cleanup; 574 } 575 576 strcpy(name, description); 577 strcpy(&name[descr_len], suffix); 578 spec->unpriv.name = name; 579 } 580 581 if (spec->mode_mask & (PRIV | UNPRIV)) { 582 if (!has_unpriv_result) 583 spec->unpriv.expect_failure = spec->priv.expect_failure; 584 585 if (!has_unpriv_retval) { 586 spec->unpriv.retval = spec->priv.retval; 587 spec->unpriv.execute = spec->priv.execute; 588 } 589 590 if (spec->unpriv.expect_msgs.cnt == 0) 591 clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs); 592 if (spec->unpriv.expect_xlated.cnt == 0) 593 clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated); 594 if (spec->unpriv.jited.cnt == 0) 595 clone_msgs(&spec->priv.jited, &spec->unpriv.jited); 596 } 597 598 spec->valid = true; 599 600 return 0; 601 602 cleanup: 603 free_test_spec(spec); 604 return err; 605 } 606 607 static void prepare_case(struct test_loader *tester, 608 struct test_spec *spec, 609 struct bpf_object *obj, 610 struct bpf_program *prog) 611 { 612 int min_log_level = 0, prog_flags; 613 614 if (env.verbosity > VERBOSE_NONE) 615 min_log_level = 1; 616 if (env.verbosity > VERBOSE_VERY) 617 min_log_level = 2; 618 619 bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz); 620 621 /* Make sure we set at least minimal log level, unless test requires 622 * even higher level already. Make sure to preserve independent log 623 * level 4 (verifier stats), though. 624 */ 625 if ((spec->log_level & 3) < min_log_level) 626 bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level); 627 else 628 bpf_program__set_log_level(prog, spec->log_level); 629 630 prog_flags = bpf_program__flags(prog); 631 bpf_program__set_flags(prog, prog_flags | spec->prog_flags); 632 633 tester->log_buf[0] = '\0'; 634 } 635 636 static void emit_verifier_log(const char *log_buf, bool force) 637 { 638 if (!force && env.verbosity == VERBOSE_NONE) 639 return; 640 fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf); 641 } 642 643 static void emit_xlated(const char *xlated, bool force) 644 { 645 if (!force && env.verbosity == VERBOSE_NONE) 646 return; 647 fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated); 648 } 649 650 static void emit_jited(const char *jited, bool force) 651 { 652 if (!force && env.verbosity == VERBOSE_NONE) 653 return; 654 fprintf(stdout, "JITED:\n=============\n%s=============\n", jited); 655 } 656 657 static void validate_msgs(char *log_buf, struct expected_msgs *msgs, 658 void (*emit_fn)(const char *buf, bool force)) 659 { 660 const char *log = log_buf, *prev_match; 661 regmatch_t reg_match[1]; 662 int prev_match_line; 663 int match_line; 664 int i, j, err; 665 666 prev_match_line = -1; 667 match_line = 0; 668 prev_match = log; 669 for (i = 0; i < msgs->cnt; i++) { 670 struct expect_msg *msg = &msgs->patterns[i]; 671 const char *match = NULL, *pat_status; 672 bool wrong_line = false; 673 674 if (!msg->is_regex) { 675 match = strstr(log, msg->substr); 676 if (match) 677 log = match + strlen(msg->substr); 678 } else { 679 err = regexec(&msg->regex, log, 1, reg_match, 0); 680 if (err == 0) { 681 match = log + reg_match[0].rm_so; 682 log += reg_match[0].rm_eo; 683 } 684 } 685 686 if (match) { 687 for (; prev_match < match; ++prev_match) 688 if (*prev_match == '\n') 689 ++match_line; 690 wrong_line = msg->on_next_line && prev_match_line >= 0 && 691 prev_match_line + 1 != match_line; 692 } 693 694 if (!match || wrong_line) { 695 PRINT_FAIL("expect_msg\n"); 696 if (env.verbosity == VERBOSE_NONE) 697 emit_fn(log_buf, true /*force*/); 698 for (j = 0; j <= i; j++) { 699 msg = &msgs->patterns[j]; 700 if (j < i) 701 pat_status = "MATCHED "; 702 else if (wrong_line) 703 pat_status = "WRONG LINE"; 704 else 705 pat_status = "EXPECTED "; 706 msg = &msgs->patterns[j]; 707 fprintf(stderr, "%s %s: '%s'\n", 708 pat_status, 709 msg->is_regex ? " REGEX" : "SUBSTR", 710 msg->substr); 711 } 712 if (wrong_line) { 713 fprintf(stderr, 714 "expecting match at line %d, actual match is at line %d\n", 715 prev_match_line + 1, match_line); 716 } 717 break; 718 } 719 720 prev_match_line = match_line; 721 } 722 } 723 724 struct cap_state { 725 __u64 old_caps; 726 bool initialized; 727 }; 728 729 static int drop_capabilities(struct cap_state *caps) 730 { 731 const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN | 732 1ULL << CAP_PERFMON | 1ULL << CAP_BPF); 733 int err; 734 735 err = cap_disable_effective(caps_to_drop, &caps->old_caps); 736 if (err) { 737 PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(err)); 738 return err; 739 } 740 741 caps->initialized = true; 742 return 0; 743 } 744 745 static int restore_capabilities(struct cap_state *caps) 746 { 747 int err; 748 749 if (!caps->initialized) 750 return 0; 751 752 err = cap_enable_effective(caps->old_caps, NULL); 753 if (err) 754 PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(err)); 755 caps->initialized = false; 756 return err; 757 } 758 759 static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec) 760 { 761 if (sysctl_unpriv_disabled < 0) 762 sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0; 763 if (sysctl_unpriv_disabled) 764 return false; 765 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) 766 return false; 767 return true; 768 } 769 770 static bool is_unpriv_capable_map(struct bpf_map *map) 771 { 772 enum bpf_map_type type; 773 __u32 flags; 774 775 type = bpf_map__type(map); 776 777 switch (type) { 778 case BPF_MAP_TYPE_HASH: 779 case BPF_MAP_TYPE_PERCPU_HASH: 780 case BPF_MAP_TYPE_HASH_OF_MAPS: 781 flags = bpf_map__map_flags(map); 782 return !(flags & BPF_F_ZERO_SEED); 783 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 784 case BPF_MAP_TYPE_ARRAY: 785 case BPF_MAP_TYPE_RINGBUF: 786 case BPF_MAP_TYPE_PROG_ARRAY: 787 case BPF_MAP_TYPE_CGROUP_ARRAY: 788 case BPF_MAP_TYPE_PERCPU_ARRAY: 789 case BPF_MAP_TYPE_USER_RINGBUF: 790 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 791 case BPF_MAP_TYPE_CGROUP_STORAGE: 792 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 793 return true; 794 default: 795 return false; 796 } 797 } 798 799 static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts) 800 { 801 __u8 tmp_out[TEST_DATA_LEN << 2] = {}; 802 __u8 tmp_in[TEST_DATA_LEN] = {}; 803 int err, saved_errno; 804 LIBBPF_OPTS(bpf_test_run_opts, topts, 805 .data_in = tmp_in, 806 .data_size_in = sizeof(tmp_in), 807 .data_out = tmp_out, 808 .data_size_out = sizeof(tmp_out), 809 .repeat = 1, 810 ); 811 812 if (empty_opts) { 813 memset(&topts, 0, sizeof(struct bpf_test_run_opts)); 814 topts.sz = sizeof(struct bpf_test_run_opts); 815 } 816 err = bpf_prog_test_run_opts(fd_prog, &topts); 817 saved_errno = errno; 818 819 if (err) { 820 PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ", 821 saved_errno, strerror(saved_errno)); 822 return err; 823 } 824 825 ASSERT_OK(0, "bpf_prog_test_run"); 826 *retval = topts.retval; 827 828 return 0; 829 } 830 831 static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec) 832 { 833 if (!subspec->execute) 834 return false; 835 836 if (subspec->expect_failure) 837 return false; 838 839 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) { 840 if (env.verbosity != VERBOSE_NONE) 841 printf("alignment prevents execution\n"); 842 return false; 843 } 844 845 return true; 846 } 847 848 /* Get a disassembly of BPF program after verifier applies all rewrites */ 849 static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz) 850 { 851 struct bpf_insn *insn_start = NULL, *insn, *insn_end; 852 __u32 insns_cnt = 0, i; 853 char buf[64]; 854 FILE *out = NULL; 855 int err; 856 857 err = get_xlated_program(prog_fd, &insn_start, &insns_cnt); 858 if (!ASSERT_OK(err, "get_xlated_program")) 859 goto out; 860 out = fmemopen(text, text_sz, "w"); 861 if (!ASSERT_OK_PTR(out, "open_memstream")) 862 goto out; 863 insn_end = insn_start + insns_cnt; 864 insn = insn_start; 865 while (insn < insn_end) { 866 i = insn - insn_start; 867 insn = disasm_insn(insn, buf, sizeof(buf)); 868 fprintf(out, "%d: %s\n", i, buf); 869 } 870 fflush(out); 871 872 out: 873 free(insn_start); 874 if (out) 875 fclose(out); 876 return err; 877 } 878 879 /* this function is forced noinline and has short generic name to look better 880 * in test_progs output (in case of a failure) 881 */ 882 static noinline 883 void run_subtest(struct test_loader *tester, 884 struct bpf_object_open_opts *open_opts, 885 const void *obj_bytes, 886 size_t obj_byte_cnt, 887 struct test_spec *specs, 888 struct test_spec *spec, 889 bool unpriv) 890 { 891 struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv; 892 struct bpf_program *tprog = NULL, *tprog_iter; 893 struct bpf_link *link, *links[32] = {}; 894 struct test_spec *spec_iter; 895 struct cap_state caps = {}; 896 struct bpf_object *tobj; 897 struct bpf_map *map; 898 int retval, err, i; 899 int links_cnt = 0; 900 bool should_load; 901 902 if (!test__start_subtest(subspec->name)) 903 return; 904 905 if ((get_current_arch() & spec->arch_mask) == 0) { 906 test__skip(); 907 return; 908 } 909 910 if (unpriv) { 911 if (!can_execute_unpriv(tester, spec)) { 912 test__skip(); 913 test__end_subtest(); 914 return; 915 } 916 if (drop_capabilities(&caps)) { 917 test__end_subtest(); 918 return; 919 } 920 } 921 922 /* Implicitly reset to NULL if next test case doesn't specify */ 923 open_opts->btf_custom_path = spec->btf_custom_path; 924 925 tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts); 926 if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */ 927 goto subtest_cleanup; 928 929 i = 0; 930 bpf_object__for_each_program(tprog_iter, tobj) { 931 spec_iter = &specs[i++]; 932 should_load = false; 933 934 if (spec_iter->valid) { 935 if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) { 936 tprog = tprog_iter; 937 should_load = true; 938 } 939 940 if (spec_iter->auxiliary && 941 spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV)) 942 should_load = true; 943 } 944 945 bpf_program__set_autoload(tprog_iter, should_load); 946 } 947 948 prepare_case(tester, spec, tobj, tprog); 949 950 /* By default bpf_object__load() automatically creates all 951 * maps declared in the skeleton. Some map types are only 952 * allowed in priv mode. Disable autoload for such maps in 953 * unpriv mode. 954 */ 955 bpf_object__for_each_map(map, tobj) 956 bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map)); 957 958 err = bpf_object__load(tobj); 959 if (subspec->expect_failure) { 960 if (!ASSERT_ERR(err, "unexpected_load_success")) { 961 emit_verifier_log(tester->log_buf, false /*force*/); 962 goto tobj_cleanup; 963 } 964 } else { 965 if (!ASSERT_OK(err, "unexpected_load_failure")) { 966 emit_verifier_log(tester->log_buf, true /*force*/); 967 goto tobj_cleanup; 968 } 969 } 970 emit_verifier_log(tester->log_buf, false /*force*/); 971 validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log); 972 973 if (subspec->expect_xlated.cnt) { 974 err = get_xlated_program_text(bpf_program__fd(tprog), 975 tester->log_buf, tester->log_buf_sz); 976 if (err) 977 goto tobj_cleanup; 978 emit_xlated(tester->log_buf, false /*force*/); 979 validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated); 980 } 981 982 if (subspec->jited.cnt) { 983 err = get_jited_program_text(bpf_program__fd(tprog), 984 tester->log_buf, tester->log_buf_sz); 985 if (err == -EOPNOTSUPP) { 986 printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__); 987 printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__); 988 test__skip(); 989 goto tobj_cleanup; 990 } 991 if (!ASSERT_EQ(err, 0, "get_jited_program_text")) 992 goto tobj_cleanup; 993 emit_jited(tester->log_buf, false /*force*/); 994 validate_msgs(tester->log_buf, &subspec->jited, emit_jited); 995 } 996 997 if (should_do_test_run(spec, subspec)) { 998 /* For some reason test_verifier executes programs 999 * with all capabilities restored. Do the same here. 1000 */ 1001 if (restore_capabilities(&caps)) 1002 goto tobj_cleanup; 1003 1004 /* Do bpf_map__attach_struct_ops() for each struct_ops map. 1005 * This should trigger bpf_struct_ops->reg callback on kernel side. 1006 */ 1007 bpf_object__for_each_map(map, tobj) { 1008 if (!bpf_map__autocreate(map) || 1009 bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS) 1010 continue; 1011 if (links_cnt >= ARRAY_SIZE(links)) { 1012 PRINT_FAIL("too many struct_ops maps"); 1013 goto tobj_cleanup; 1014 } 1015 link = bpf_map__attach_struct_ops(map); 1016 if (!link) { 1017 PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n", 1018 bpf_map__name(map), err); 1019 goto tobj_cleanup; 1020 } 1021 links[links_cnt++] = link; 1022 } 1023 1024 if (tester->pre_execution_cb) { 1025 err = tester->pre_execution_cb(tobj); 1026 if (err) { 1027 PRINT_FAIL("pre_execution_cb failed: %d\n", err); 1028 goto tobj_cleanup; 1029 } 1030 } 1031 1032 do_prog_test_run(bpf_program__fd(tprog), &retval, 1033 bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false); 1034 if (retval != subspec->retval && subspec->retval != POINTER_VALUE) { 1035 PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval); 1036 goto tobj_cleanup; 1037 } 1038 /* redo bpf_map__attach_struct_ops for each test */ 1039 while (links_cnt > 0) 1040 bpf_link__destroy(links[--links_cnt]); 1041 } 1042 1043 tobj_cleanup: 1044 while (links_cnt > 0) 1045 bpf_link__destroy(links[--links_cnt]); 1046 bpf_object__close(tobj); 1047 subtest_cleanup: 1048 test__end_subtest(); 1049 restore_capabilities(&caps); 1050 } 1051 1052 static void process_subtest(struct test_loader *tester, 1053 const char *skel_name, 1054 skel_elf_bytes_fn elf_bytes_factory) 1055 { 1056 LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name); 1057 struct test_spec *specs = NULL; 1058 struct bpf_object *obj = NULL; 1059 struct bpf_program *prog; 1060 const void *obj_bytes; 1061 int err, i, nr_progs; 1062 size_t obj_byte_cnt; 1063 1064 if (tester_init(tester) < 0) 1065 return; /* failed to initialize tester */ 1066 1067 obj_bytes = elf_bytes_factory(&obj_byte_cnt); 1068 obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts); 1069 if (!ASSERT_OK_PTR(obj, "obj_open_mem")) 1070 return; 1071 1072 nr_progs = 0; 1073 bpf_object__for_each_program(prog, obj) 1074 ++nr_progs; 1075 1076 specs = calloc(nr_progs, sizeof(struct test_spec)); 1077 if (!ASSERT_OK_PTR(specs, "specs_alloc")) 1078 return; 1079 1080 i = 0; 1081 bpf_object__for_each_program(prog, obj) { 1082 /* ignore tests for which we can't derive test specification */ 1083 err = parse_test_spec(tester, obj, prog, &specs[i++]); 1084 if (err) 1085 PRINT_FAIL("Can't parse test spec for program '%s'\n", 1086 bpf_program__name(prog)); 1087 } 1088 1089 i = 0; 1090 bpf_object__for_each_program(prog, obj) { 1091 struct test_spec *spec = &specs[i++]; 1092 1093 if (!spec->valid || spec->auxiliary) 1094 continue; 1095 1096 if (spec->mode_mask & PRIV) 1097 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt, 1098 specs, spec, false); 1099 if (spec->mode_mask & UNPRIV) 1100 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt, 1101 specs, spec, true); 1102 1103 } 1104 1105 for (i = 0; i < nr_progs; ++i) 1106 free_test_spec(&specs[i]); 1107 free(specs); 1108 bpf_object__close(obj); 1109 } 1110 1111 void test_loader__run_subtests(struct test_loader *tester, 1112 const char *skel_name, 1113 skel_elf_bytes_fn elf_bytes_factory) 1114 { 1115 /* see comment in run_subtest() for why we do this function nesting */ 1116 process_subtest(tester, skel_name, elf_bytes_factory); 1117 } 1118