1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/capability.h> 4 #include <stdlib.h> 5 #include <regex.h> 6 #include <test_progs.h> 7 #include <bpf/btf.h> 8 9 #include "autoconf_helper.h" 10 #include "disasm_helpers.h" 11 #include "unpriv_helpers.h" 12 #include "cap_helpers.h" 13 #include "jit_disasm_helpers.h" 14 15 #define str_has_pfx(str, pfx) \ 16 (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0) 17 18 #define TEST_LOADER_LOG_BUF_SZ 2097152 19 20 #define TEST_TAG_EXPECT_FAILURE "comment:test_expect_failure" 21 #define TEST_TAG_EXPECT_SUCCESS "comment:test_expect_success" 22 #define TEST_TAG_EXPECT_MSG_PFX "comment:test_expect_msg=" 23 #define TEST_TAG_EXPECT_XLATED_PFX "comment:test_expect_xlated=" 24 #define TEST_TAG_EXPECT_FAILURE_UNPRIV "comment:test_expect_failure_unpriv" 25 #define TEST_TAG_EXPECT_SUCCESS_UNPRIV "comment:test_expect_success_unpriv" 26 #define TEST_TAG_EXPECT_MSG_PFX_UNPRIV "comment:test_expect_msg_unpriv=" 27 #define TEST_TAG_EXPECT_XLATED_PFX_UNPRIV "comment:test_expect_xlated_unpriv=" 28 #define TEST_TAG_LOG_LEVEL_PFX "comment:test_log_level=" 29 #define TEST_TAG_PROG_FLAGS_PFX "comment:test_prog_flags=" 30 #define TEST_TAG_DESCRIPTION_PFX "comment:test_description=" 31 #define TEST_TAG_RETVAL_PFX "comment:test_retval=" 32 #define TEST_TAG_RETVAL_PFX_UNPRIV "comment:test_retval_unpriv=" 33 #define TEST_TAG_AUXILIARY "comment:test_auxiliary" 34 #define TEST_TAG_AUXILIARY_UNPRIV "comment:test_auxiliary_unpriv" 35 #define TEST_BTF_PATH "comment:test_btf_path=" 36 #define TEST_TAG_ARCH "comment:test_arch=" 37 #define TEST_TAG_JITED_PFX "comment:test_jited=" 38 #define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv=" 39 #define TEST_TAG_CAPS_UNPRIV "comment:test_caps_unpriv=" 40 41 /* Warning: duplicated in bpf_misc.h */ 42 #define POINTER_VALUE 0xcafe4all 43 #define TEST_DATA_LEN 64 44 45 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 46 #define EFFICIENT_UNALIGNED_ACCESS 1 47 #else 48 #define EFFICIENT_UNALIGNED_ACCESS 0 49 #endif 50 51 static int sysctl_unpriv_disabled = -1; 52 53 enum mode { 54 PRIV = 1, 55 UNPRIV = 2 56 }; 57 58 struct expect_msg { 59 const char *substr; /* substring match */ 60 regex_t regex; 61 bool is_regex; 62 bool on_next_line; 63 }; 64 65 struct expected_msgs { 66 struct expect_msg *patterns; 67 size_t cnt; 68 }; 69 70 struct test_subspec { 71 char *name; 72 bool expect_failure; 73 struct expected_msgs expect_msgs; 74 struct expected_msgs expect_xlated; 75 struct expected_msgs jited; 76 int retval; 77 bool execute; 78 __u64 caps; 79 }; 80 81 struct test_spec { 82 const char *prog_name; 83 struct test_subspec priv; 84 struct test_subspec unpriv; 85 const char *btf_custom_path; 86 int log_level; 87 int prog_flags; 88 int mode_mask; 89 int arch_mask; 90 bool auxiliary; 91 bool valid; 92 }; 93 94 static int tester_init(struct test_loader *tester) 95 { 96 if (!tester->log_buf) { 97 tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ; 98 tester->log_buf = calloc(tester->log_buf_sz, 1); 99 if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf")) 100 return -ENOMEM; 101 } 102 103 return 0; 104 } 105 106 void test_loader_fini(struct test_loader *tester) 107 { 108 if (!tester) 109 return; 110 111 free(tester->log_buf); 112 } 113 114 static void free_msgs(struct expected_msgs *msgs) 115 { 116 int i; 117 118 for (i = 0; i < msgs->cnt; i++) 119 if (msgs->patterns[i].is_regex) 120 regfree(&msgs->patterns[i].regex); 121 free(msgs->patterns); 122 msgs->patterns = NULL; 123 msgs->cnt = 0; 124 } 125 126 static void free_test_spec(struct test_spec *spec) 127 { 128 /* Deallocate expect_msgs arrays. */ 129 free_msgs(&spec->priv.expect_msgs); 130 free_msgs(&spec->unpriv.expect_msgs); 131 free_msgs(&spec->priv.expect_xlated); 132 free_msgs(&spec->unpriv.expect_xlated); 133 free_msgs(&spec->priv.jited); 134 free_msgs(&spec->unpriv.jited); 135 136 free(spec->priv.name); 137 free(spec->unpriv.name); 138 spec->priv.name = NULL; 139 spec->unpriv.name = NULL; 140 } 141 142 /* Compiles regular expression matching pattern. 143 * Pattern has a special syntax: 144 * 145 * pattern := (<verbatim text> | regex)* 146 * regex := "{{" <posix extended regular expression> "}}" 147 * 148 * In other words, pattern is a verbatim text with inclusion 149 * of regular expressions enclosed in "{{" "}}" pairs. 150 * For example, pattern "foo{{[0-9]+}}" matches strings like 151 * "foo0", "foo007", etc. 152 */ 153 static int compile_regex(const char *pattern, regex_t *regex) 154 { 155 char err_buf[256], buf[256] = {}, *ptr, *buf_end; 156 const char *original_pattern = pattern; 157 bool in_regex = false; 158 int err; 159 160 buf_end = buf + sizeof(buf); 161 ptr = buf; 162 while (*pattern && ptr < buf_end - 2) { 163 if (!in_regex && str_has_pfx(pattern, "{{")) { 164 in_regex = true; 165 pattern += 2; 166 continue; 167 } 168 if (in_regex && str_has_pfx(pattern, "}}")) { 169 in_regex = false; 170 pattern += 2; 171 continue; 172 } 173 if (in_regex) { 174 *ptr++ = *pattern++; 175 continue; 176 } 177 /* list of characters that need escaping for extended posix regex */ 178 if (strchr(".[]\\()*+?{}|^$", *pattern)) { 179 *ptr++ = '\\'; 180 *ptr++ = *pattern++; 181 continue; 182 } 183 *ptr++ = *pattern++; 184 } 185 if (*pattern) { 186 PRINT_FAIL("Regexp too long: '%s'\n", original_pattern); 187 return -EINVAL; 188 } 189 if (in_regex) { 190 PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern); 191 return -EINVAL; 192 } 193 err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE); 194 if (err != 0) { 195 regerror(err, regex, err_buf, sizeof(err_buf)); 196 PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf); 197 return -EINVAL; 198 } 199 return 0; 200 } 201 202 static int __push_msg(const char *pattern, bool on_next_line, struct expected_msgs *msgs) 203 { 204 struct expect_msg *msg; 205 void *tmp; 206 int err; 207 208 tmp = realloc(msgs->patterns, 209 (1 + msgs->cnt) * sizeof(struct expect_msg)); 210 if (!tmp) { 211 ASSERT_FAIL("failed to realloc memory for messages\n"); 212 return -ENOMEM; 213 } 214 msgs->patterns = tmp; 215 msg = &msgs->patterns[msgs->cnt]; 216 msg->on_next_line = on_next_line; 217 msg->substr = pattern; 218 msg->is_regex = false; 219 if (strstr(pattern, "{{")) { 220 err = compile_regex(pattern, &msg->regex); 221 if (err) 222 return err; 223 msg->is_regex = true; 224 } 225 msgs->cnt += 1; 226 return 0; 227 } 228 229 static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to) 230 { 231 struct expect_msg *msg; 232 int i, err; 233 234 for (i = 0; i < from->cnt; i++) { 235 msg = &from->patterns[i]; 236 err = __push_msg(msg->substr, msg->on_next_line, to); 237 if (err) 238 return err; 239 } 240 return 0; 241 } 242 243 static int push_msg(const char *substr, struct expected_msgs *msgs) 244 { 245 return __push_msg(substr, false, msgs); 246 } 247 248 static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs) 249 { 250 int err; 251 252 if (strcmp(regex_str, "...") == 0) { 253 *on_next_line = false; 254 return 0; 255 } 256 err = __push_msg(regex_str, *on_next_line, msgs); 257 if (err) 258 return err; 259 *on_next_line = true; 260 return 0; 261 } 262 263 static int parse_int(const char *str, int *val, const char *name) 264 { 265 char *end; 266 long tmp; 267 268 errno = 0; 269 if (str_has_pfx(str, "0x")) 270 tmp = strtol(str + 2, &end, 16); 271 else 272 tmp = strtol(str, &end, 10); 273 if (errno || end[0] != '\0') { 274 PRINT_FAIL("failed to parse %s from '%s'\n", name, str); 275 return -EINVAL; 276 } 277 *val = tmp; 278 return 0; 279 } 280 281 static int parse_caps(const char *str, __u64 *val, const char *name) 282 { 283 int cap_flag = 0; 284 char *token = NULL, *saveptr = NULL; 285 286 char *str_cpy = strdup(str); 287 if (str_cpy == NULL) { 288 PRINT_FAIL("Memory allocation failed\n"); 289 return -EINVAL; 290 } 291 292 token = strtok_r(str_cpy, "|", &saveptr); 293 while (token != NULL) { 294 errno = 0; 295 if (!strncmp("CAP_", token, sizeof("CAP_") - 1)) { 296 PRINT_FAIL("define %s constant in bpf_misc.h, failed to parse caps\n", token); 297 return -EINVAL; 298 } 299 cap_flag = strtol(token, NULL, 10); 300 if (!cap_flag || errno) { 301 PRINT_FAIL("failed to parse caps %s\n", name); 302 return -EINVAL; 303 } 304 *val |= (1ULL << cap_flag); 305 token = strtok_r(NULL, "|", &saveptr); 306 } 307 308 free(str_cpy); 309 return 0; 310 } 311 312 static int parse_retval(const char *str, int *val, const char *name) 313 { 314 struct { 315 char *name; 316 int val; 317 } named_values[] = { 318 { "INT_MIN" , INT_MIN }, 319 { "POINTER_VALUE", POINTER_VALUE }, 320 { "TEST_DATA_LEN", TEST_DATA_LEN }, 321 }; 322 int i; 323 324 for (i = 0; i < ARRAY_SIZE(named_values); ++i) { 325 if (strcmp(str, named_values[i].name) != 0) 326 continue; 327 *val = named_values[i].val; 328 return 0; 329 } 330 331 return parse_int(str, val, name); 332 } 333 334 static void update_flags(int *flags, int flag, bool clear) 335 { 336 if (clear) 337 *flags &= ~flag; 338 else 339 *flags |= flag; 340 } 341 342 /* Matches a string of form '<pfx>[^=]=.*' and returns it's suffix. 343 * Used to parse btf_decl_tag values. 344 * Such values require unique prefix because compiler does not add 345 * same __attribute__((btf_decl_tag(...))) twice. 346 * Test suite uses two-component tags for such cases: 347 * 348 * <pfx> __COUNTER__ '=' 349 * 350 * For example, two consecutive __msg tags '__msg("foo") __msg("foo")' 351 * would be encoded as: 352 * 353 * [18] DECL_TAG 'comment:test_expect_msg=0=foo' type_id=15 component_idx=-1 354 * [19] DECL_TAG 'comment:test_expect_msg=1=foo' type_id=15 component_idx=-1 355 * 356 * And the purpose of this function is to extract 'foo' from the above. 357 */ 358 static const char *skip_dynamic_pfx(const char *s, const char *pfx) 359 { 360 const char *msg; 361 362 if (strncmp(s, pfx, strlen(pfx)) != 0) 363 return NULL; 364 msg = s + strlen(pfx); 365 msg = strchr(msg, '='); 366 if (!msg) 367 return NULL; 368 return msg + 1; 369 } 370 371 enum arch { 372 ARCH_UNKNOWN = 0x1, 373 ARCH_X86_64 = 0x2, 374 ARCH_ARM64 = 0x4, 375 ARCH_RISCV64 = 0x8, 376 }; 377 378 static int get_current_arch(void) 379 { 380 #if defined(__x86_64__) 381 return ARCH_X86_64; 382 #elif defined(__aarch64__) 383 return ARCH_ARM64; 384 #elif defined(__riscv) && __riscv_xlen == 64 385 return ARCH_RISCV64; 386 #endif 387 return ARCH_UNKNOWN; 388 } 389 390 /* Uses btf_decl_tag attributes to describe the expected test 391 * behavior, see bpf_misc.h for detailed description of each attribute 392 * and attribute combinations. 393 */ 394 static int parse_test_spec(struct test_loader *tester, 395 struct bpf_object *obj, 396 struct bpf_program *prog, 397 struct test_spec *spec) 398 { 399 const char *description = NULL; 400 bool has_unpriv_result = false; 401 bool has_unpriv_retval = false; 402 bool unpriv_xlated_on_next_line = true; 403 bool xlated_on_next_line = true; 404 bool unpriv_jit_on_next_line; 405 bool jit_on_next_line; 406 bool collect_jit = false; 407 int func_id, i, err = 0; 408 u32 arch_mask = 0; 409 struct btf *btf; 410 enum arch arch; 411 412 memset(spec, 0, sizeof(*spec)); 413 414 spec->prog_name = bpf_program__name(prog); 415 spec->prog_flags = testing_prog_flags(); 416 417 btf = bpf_object__btf(obj); 418 if (!btf) { 419 ASSERT_FAIL("BPF object has no BTF"); 420 return -EINVAL; 421 } 422 423 func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC); 424 if (func_id < 0) { 425 ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name); 426 return -EINVAL; 427 } 428 429 for (i = 1; i < btf__type_cnt(btf); i++) { 430 const char *s, *val, *msg; 431 const struct btf_type *t; 432 bool clear; 433 int flags; 434 435 t = btf__type_by_id(btf, i); 436 if (!btf_is_decl_tag(t)) 437 continue; 438 439 if (t->type != func_id || btf_decl_tag(t)->component_idx != -1) 440 continue; 441 442 s = btf__str_by_offset(btf, t->name_off); 443 if (str_has_pfx(s, TEST_TAG_DESCRIPTION_PFX)) { 444 description = s + sizeof(TEST_TAG_DESCRIPTION_PFX) - 1; 445 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE) == 0) { 446 spec->priv.expect_failure = true; 447 spec->mode_mask |= PRIV; 448 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS) == 0) { 449 spec->priv.expect_failure = false; 450 spec->mode_mask |= PRIV; 451 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE_UNPRIV) == 0) { 452 spec->unpriv.expect_failure = true; 453 spec->mode_mask |= UNPRIV; 454 has_unpriv_result = true; 455 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS_UNPRIV) == 0) { 456 spec->unpriv.expect_failure = false; 457 spec->mode_mask |= UNPRIV; 458 has_unpriv_result = true; 459 } else if (strcmp(s, TEST_TAG_AUXILIARY) == 0) { 460 spec->auxiliary = true; 461 spec->mode_mask |= PRIV; 462 } else if (strcmp(s, TEST_TAG_AUXILIARY_UNPRIV) == 0) { 463 spec->auxiliary = true; 464 spec->mode_mask |= UNPRIV; 465 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX))) { 466 err = push_msg(msg, &spec->priv.expect_msgs); 467 if (err) 468 goto cleanup; 469 spec->mode_mask |= PRIV; 470 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX_UNPRIV))) { 471 err = push_msg(msg, &spec->unpriv.expect_msgs); 472 if (err) 473 goto cleanup; 474 spec->mode_mask |= UNPRIV; 475 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX))) { 476 if (arch_mask == 0) { 477 PRINT_FAIL("__jited used before __arch_*"); 478 goto cleanup; 479 } 480 if (collect_jit) { 481 err = push_disasm_msg(msg, &jit_on_next_line, 482 &spec->priv.jited); 483 if (err) 484 goto cleanup; 485 spec->mode_mask |= PRIV; 486 } 487 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX_UNPRIV))) { 488 if (arch_mask == 0) { 489 PRINT_FAIL("__unpriv_jited used before __arch_*"); 490 goto cleanup; 491 } 492 if (collect_jit) { 493 err = push_disasm_msg(msg, &unpriv_jit_on_next_line, 494 &spec->unpriv.jited); 495 if (err) 496 goto cleanup; 497 spec->mode_mask |= UNPRIV; 498 } 499 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX))) { 500 err = push_disasm_msg(msg, &xlated_on_next_line, 501 &spec->priv.expect_xlated); 502 if (err) 503 goto cleanup; 504 spec->mode_mask |= PRIV; 505 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX_UNPRIV))) { 506 err = push_disasm_msg(msg, &unpriv_xlated_on_next_line, 507 &spec->unpriv.expect_xlated); 508 if (err) 509 goto cleanup; 510 spec->mode_mask |= UNPRIV; 511 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX)) { 512 val = s + sizeof(TEST_TAG_RETVAL_PFX) - 1; 513 err = parse_retval(val, &spec->priv.retval, "__retval"); 514 if (err) 515 goto cleanup; 516 spec->priv.execute = true; 517 spec->mode_mask |= PRIV; 518 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX_UNPRIV)) { 519 val = s + sizeof(TEST_TAG_RETVAL_PFX_UNPRIV) - 1; 520 err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv"); 521 if (err) 522 goto cleanup; 523 spec->mode_mask |= UNPRIV; 524 spec->unpriv.execute = true; 525 has_unpriv_retval = true; 526 } else if (str_has_pfx(s, TEST_TAG_LOG_LEVEL_PFX)) { 527 val = s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1; 528 err = parse_int(val, &spec->log_level, "test log level"); 529 if (err) 530 goto cleanup; 531 } else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) { 532 val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1; 533 534 clear = val[0] == '!'; 535 if (clear) 536 val++; 537 538 if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) { 539 update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear); 540 } else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) { 541 update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear); 542 } else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) { 543 update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear); 544 } else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) { 545 update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear); 546 } else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) { 547 update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear); 548 } else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) { 549 update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); 550 } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { 551 update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); 552 } else /* assume numeric value */ { 553 err = parse_int(val, &flags, "test prog flags"); 554 if (err) 555 goto cleanup; 556 update_flags(&spec->prog_flags, flags, clear); 557 } 558 } else if (str_has_pfx(s, TEST_TAG_ARCH)) { 559 val = s + sizeof(TEST_TAG_ARCH) - 1; 560 if (strcmp(val, "X86_64") == 0) { 561 arch = ARCH_X86_64; 562 } else if (strcmp(val, "ARM64") == 0) { 563 arch = ARCH_ARM64; 564 } else if (strcmp(val, "RISCV64") == 0) { 565 arch = ARCH_RISCV64; 566 } else { 567 PRINT_FAIL("bad arch spec: '%s'", val); 568 err = -EINVAL; 569 goto cleanup; 570 } 571 arch_mask |= arch; 572 collect_jit = get_current_arch() == arch; 573 unpriv_jit_on_next_line = true; 574 jit_on_next_line = true; 575 } else if (str_has_pfx(s, TEST_BTF_PATH)) { 576 spec->btf_custom_path = s + sizeof(TEST_BTF_PATH) - 1; 577 } else if (str_has_pfx(s, TEST_TAG_CAPS_UNPRIV)) { 578 val = s + sizeof(TEST_TAG_CAPS_UNPRIV) - 1; 579 err = parse_caps(val, &spec->unpriv.caps, "test caps"); 580 if (err) 581 goto cleanup; 582 spec->mode_mask |= UNPRIV; 583 } 584 } 585 586 spec->arch_mask = arch_mask ?: -1; 587 588 if (spec->mode_mask == 0) 589 spec->mode_mask = PRIV; 590 591 if (!description) 592 description = spec->prog_name; 593 594 if (spec->mode_mask & PRIV) { 595 spec->priv.name = strdup(description); 596 if (!spec->priv.name) { 597 PRINT_FAIL("failed to allocate memory for priv.name\n"); 598 err = -ENOMEM; 599 goto cleanup; 600 } 601 } 602 603 if (spec->mode_mask & UNPRIV) { 604 int descr_len = strlen(description); 605 const char *suffix = " @unpriv"; 606 char *name; 607 608 name = malloc(descr_len + strlen(suffix) + 1); 609 if (!name) { 610 PRINT_FAIL("failed to allocate memory for unpriv.name\n"); 611 err = -ENOMEM; 612 goto cleanup; 613 } 614 615 strcpy(name, description); 616 strcpy(&name[descr_len], suffix); 617 spec->unpriv.name = name; 618 } 619 620 if (spec->mode_mask & (PRIV | UNPRIV)) { 621 if (!has_unpriv_result) 622 spec->unpriv.expect_failure = spec->priv.expect_failure; 623 624 if (!has_unpriv_retval) { 625 spec->unpriv.retval = spec->priv.retval; 626 spec->unpriv.execute = spec->priv.execute; 627 } 628 629 if (spec->unpriv.expect_msgs.cnt == 0) 630 clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs); 631 if (spec->unpriv.expect_xlated.cnt == 0) 632 clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated); 633 if (spec->unpriv.jited.cnt == 0) 634 clone_msgs(&spec->priv.jited, &spec->unpriv.jited); 635 } 636 637 spec->valid = true; 638 639 return 0; 640 641 cleanup: 642 free_test_spec(spec); 643 return err; 644 } 645 646 static void prepare_case(struct test_loader *tester, 647 struct test_spec *spec, 648 struct bpf_object *obj, 649 struct bpf_program *prog) 650 { 651 int min_log_level = 0, prog_flags; 652 653 if (env.verbosity > VERBOSE_NONE) 654 min_log_level = 1; 655 if (env.verbosity > VERBOSE_VERY) 656 min_log_level = 2; 657 658 bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz); 659 660 /* Make sure we set at least minimal log level, unless test requires 661 * even higher level already. Make sure to preserve independent log 662 * level 4 (verifier stats), though. 663 */ 664 if ((spec->log_level & 3) < min_log_level) 665 bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level); 666 else 667 bpf_program__set_log_level(prog, spec->log_level); 668 669 prog_flags = bpf_program__flags(prog); 670 bpf_program__set_flags(prog, prog_flags | spec->prog_flags); 671 672 tester->log_buf[0] = '\0'; 673 } 674 675 static void emit_verifier_log(const char *log_buf, bool force) 676 { 677 if (!force && env.verbosity == VERBOSE_NONE) 678 return; 679 fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf); 680 } 681 682 static void emit_xlated(const char *xlated, bool force) 683 { 684 if (!force && env.verbosity == VERBOSE_NONE) 685 return; 686 fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated); 687 } 688 689 static void emit_jited(const char *jited, bool force) 690 { 691 if (!force && env.verbosity == VERBOSE_NONE) 692 return; 693 fprintf(stdout, "JITED:\n=============\n%s=============\n", jited); 694 } 695 696 static void validate_msgs(char *log_buf, struct expected_msgs *msgs, 697 void (*emit_fn)(const char *buf, bool force)) 698 { 699 const char *log = log_buf, *prev_match; 700 regmatch_t reg_match[1]; 701 int prev_match_line; 702 int match_line; 703 int i, j, err; 704 705 prev_match_line = -1; 706 match_line = 0; 707 prev_match = log; 708 for (i = 0; i < msgs->cnt; i++) { 709 struct expect_msg *msg = &msgs->patterns[i]; 710 const char *match = NULL, *pat_status; 711 bool wrong_line = false; 712 713 if (!msg->is_regex) { 714 match = strstr(log, msg->substr); 715 if (match) 716 log = match + strlen(msg->substr); 717 } else { 718 err = regexec(&msg->regex, log, 1, reg_match, 0); 719 if (err == 0) { 720 match = log + reg_match[0].rm_so; 721 log += reg_match[0].rm_eo; 722 } 723 } 724 725 if (match) { 726 for (; prev_match < match; ++prev_match) 727 if (*prev_match == '\n') 728 ++match_line; 729 wrong_line = msg->on_next_line && prev_match_line >= 0 && 730 prev_match_line + 1 != match_line; 731 } 732 733 if (!match || wrong_line) { 734 PRINT_FAIL("expect_msg\n"); 735 if (env.verbosity == VERBOSE_NONE) 736 emit_fn(log_buf, true /*force*/); 737 for (j = 0; j <= i; j++) { 738 msg = &msgs->patterns[j]; 739 if (j < i) 740 pat_status = "MATCHED "; 741 else if (wrong_line) 742 pat_status = "WRONG LINE"; 743 else 744 pat_status = "EXPECTED "; 745 msg = &msgs->patterns[j]; 746 fprintf(stderr, "%s %s: '%s'\n", 747 pat_status, 748 msg->is_regex ? " REGEX" : "SUBSTR", 749 msg->substr); 750 } 751 if (wrong_line) { 752 fprintf(stderr, 753 "expecting match at line %d, actual match is at line %d\n", 754 prev_match_line + 1, match_line); 755 } 756 break; 757 } 758 759 prev_match_line = match_line; 760 } 761 } 762 763 struct cap_state { 764 __u64 old_caps; 765 bool initialized; 766 }; 767 768 static int drop_capabilities(struct cap_state *caps) 769 { 770 const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN | 771 1ULL << CAP_PERFMON | 1ULL << CAP_BPF); 772 int err; 773 774 err = cap_disable_effective(caps_to_drop, &caps->old_caps); 775 if (err) { 776 PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(err)); 777 return err; 778 } 779 780 caps->initialized = true; 781 return 0; 782 } 783 784 static int restore_capabilities(struct cap_state *caps) 785 { 786 int err; 787 788 if (!caps->initialized) 789 return 0; 790 791 err = cap_enable_effective(caps->old_caps, NULL); 792 if (err) 793 PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(err)); 794 caps->initialized = false; 795 return err; 796 } 797 798 static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec) 799 { 800 if (sysctl_unpriv_disabled < 0) 801 sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0; 802 if (sysctl_unpriv_disabled) 803 return false; 804 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) 805 return false; 806 return true; 807 } 808 809 static bool is_unpriv_capable_map(struct bpf_map *map) 810 { 811 enum bpf_map_type type; 812 __u32 flags; 813 814 type = bpf_map__type(map); 815 816 switch (type) { 817 case BPF_MAP_TYPE_HASH: 818 case BPF_MAP_TYPE_PERCPU_HASH: 819 case BPF_MAP_TYPE_HASH_OF_MAPS: 820 flags = bpf_map__map_flags(map); 821 return !(flags & BPF_F_ZERO_SEED); 822 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 823 case BPF_MAP_TYPE_ARRAY: 824 case BPF_MAP_TYPE_RINGBUF: 825 case BPF_MAP_TYPE_PROG_ARRAY: 826 case BPF_MAP_TYPE_CGROUP_ARRAY: 827 case BPF_MAP_TYPE_PERCPU_ARRAY: 828 case BPF_MAP_TYPE_USER_RINGBUF: 829 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 830 case BPF_MAP_TYPE_CGROUP_STORAGE: 831 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 832 return true; 833 default: 834 return false; 835 } 836 } 837 838 static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts) 839 { 840 __u8 tmp_out[TEST_DATA_LEN << 2] = {}; 841 __u8 tmp_in[TEST_DATA_LEN] = {}; 842 int err, saved_errno; 843 LIBBPF_OPTS(bpf_test_run_opts, topts, 844 .data_in = tmp_in, 845 .data_size_in = sizeof(tmp_in), 846 .data_out = tmp_out, 847 .data_size_out = sizeof(tmp_out), 848 .repeat = 1, 849 ); 850 851 if (empty_opts) { 852 memset(&topts, 0, sizeof(struct bpf_test_run_opts)); 853 topts.sz = sizeof(struct bpf_test_run_opts); 854 } 855 err = bpf_prog_test_run_opts(fd_prog, &topts); 856 saved_errno = errno; 857 858 if (err) { 859 PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ", 860 saved_errno, strerror(saved_errno)); 861 return err; 862 } 863 864 ASSERT_OK(0, "bpf_prog_test_run"); 865 *retval = topts.retval; 866 867 return 0; 868 } 869 870 static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec) 871 { 872 if (!subspec->execute) 873 return false; 874 875 if (subspec->expect_failure) 876 return false; 877 878 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) { 879 if (env.verbosity != VERBOSE_NONE) 880 printf("alignment prevents execution\n"); 881 return false; 882 } 883 884 return true; 885 } 886 887 /* Get a disassembly of BPF program after verifier applies all rewrites */ 888 static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz) 889 { 890 struct bpf_insn *insn_start = NULL, *insn, *insn_end; 891 __u32 insns_cnt = 0, i; 892 char buf[64]; 893 FILE *out = NULL; 894 int err; 895 896 err = get_xlated_program(prog_fd, &insn_start, &insns_cnt); 897 if (!ASSERT_OK(err, "get_xlated_program")) 898 goto out; 899 out = fmemopen(text, text_sz, "w"); 900 if (!ASSERT_OK_PTR(out, "open_memstream")) 901 goto out; 902 insn_end = insn_start + insns_cnt; 903 insn = insn_start; 904 while (insn < insn_end) { 905 i = insn - insn_start; 906 insn = disasm_insn(insn, buf, sizeof(buf)); 907 fprintf(out, "%d: %s\n", i, buf); 908 } 909 fflush(out); 910 911 out: 912 free(insn_start); 913 if (out) 914 fclose(out); 915 return err; 916 } 917 918 /* this function is forced noinline and has short generic name to look better 919 * in test_progs output (in case of a failure) 920 */ 921 static noinline 922 void run_subtest(struct test_loader *tester, 923 struct bpf_object_open_opts *open_opts, 924 const void *obj_bytes, 925 size_t obj_byte_cnt, 926 struct test_spec *specs, 927 struct test_spec *spec, 928 bool unpriv) 929 { 930 struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv; 931 struct bpf_program *tprog = NULL, *tprog_iter; 932 struct bpf_link *link, *links[32] = {}; 933 struct test_spec *spec_iter; 934 struct cap_state caps = {}; 935 struct bpf_object *tobj; 936 struct bpf_map *map; 937 int retval, err, i; 938 int links_cnt = 0; 939 bool should_load; 940 941 if (!test__start_subtest(subspec->name)) 942 return; 943 944 if ((get_current_arch() & spec->arch_mask) == 0) { 945 test__skip(); 946 return; 947 } 948 949 if (unpriv) { 950 if (!can_execute_unpriv(tester, spec)) { 951 test__skip(); 952 test__end_subtest(); 953 return; 954 } 955 if (drop_capabilities(&caps)) { 956 test__end_subtest(); 957 return; 958 } 959 if (subspec->caps) { 960 err = cap_enable_effective(subspec->caps, NULL); 961 if (err) { 962 PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(err)); 963 goto subtest_cleanup; 964 } 965 } 966 } 967 968 /* Implicitly reset to NULL if next test case doesn't specify */ 969 open_opts->btf_custom_path = spec->btf_custom_path; 970 971 tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts); 972 if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */ 973 goto subtest_cleanup; 974 975 i = 0; 976 bpf_object__for_each_program(tprog_iter, tobj) { 977 spec_iter = &specs[i++]; 978 should_load = false; 979 980 if (spec_iter->valid) { 981 if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) { 982 tprog = tprog_iter; 983 should_load = true; 984 } 985 986 if (spec_iter->auxiliary && 987 spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV)) 988 should_load = true; 989 } 990 991 bpf_program__set_autoload(tprog_iter, should_load); 992 } 993 994 prepare_case(tester, spec, tobj, tprog); 995 996 /* By default bpf_object__load() automatically creates all 997 * maps declared in the skeleton. Some map types are only 998 * allowed in priv mode. Disable autoload for such maps in 999 * unpriv mode. 1000 */ 1001 bpf_object__for_each_map(map, tobj) 1002 bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map)); 1003 1004 err = bpf_object__load(tobj); 1005 if (subspec->expect_failure) { 1006 if (!ASSERT_ERR(err, "unexpected_load_success")) { 1007 emit_verifier_log(tester->log_buf, false /*force*/); 1008 goto tobj_cleanup; 1009 } 1010 } else { 1011 if (!ASSERT_OK(err, "unexpected_load_failure")) { 1012 emit_verifier_log(tester->log_buf, true /*force*/); 1013 goto tobj_cleanup; 1014 } 1015 } 1016 emit_verifier_log(tester->log_buf, false /*force*/); 1017 validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log); 1018 1019 if (subspec->expect_xlated.cnt) { 1020 err = get_xlated_program_text(bpf_program__fd(tprog), 1021 tester->log_buf, tester->log_buf_sz); 1022 if (err) 1023 goto tobj_cleanup; 1024 emit_xlated(tester->log_buf, false /*force*/); 1025 validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated); 1026 } 1027 1028 if (subspec->jited.cnt) { 1029 err = get_jited_program_text(bpf_program__fd(tprog), 1030 tester->log_buf, tester->log_buf_sz); 1031 if (err == -EOPNOTSUPP) { 1032 printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__); 1033 printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__); 1034 test__skip(); 1035 goto tobj_cleanup; 1036 } 1037 if (!ASSERT_EQ(err, 0, "get_jited_program_text")) 1038 goto tobj_cleanup; 1039 emit_jited(tester->log_buf, false /*force*/); 1040 validate_msgs(tester->log_buf, &subspec->jited, emit_jited); 1041 } 1042 1043 if (should_do_test_run(spec, subspec)) { 1044 /* For some reason test_verifier executes programs 1045 * with all capabilities restored. Do the same here. 1046 */ 1047 if (restore_capabilities(&caps)) 1048 goto tobj_cleanup; 1049 1050 /* Do bpf_map__attach_struct_ops() for each struct_ops map. 1051 * This should trigger bpf_struct_ops->reg callback on kernel side. 1052 */ 1053 bpf_object__for_each_map(map, tobj) { 1054 if (!bpf_map__autocreate(map) || 1055 bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS) 1056 continue; 1057 if (links_cnt >= ARRAY_SIZE(links)) { 1058 PRINT_FAIL("too many struct_ops maps"); 1059 goto tobj_cleanup; 1060 } 1061 link = bpf_map__attach_struct_ops(map); 1062 if (!link) { 1063 PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n", 1064 bpf_map__name(map), err); 1065 goto tobj_cleanup; 1066 } 1067 links[links_cnt++] = link; 1068 } 1069 1070 if (tester->pre_execution_cb) { 1071 err = tester->pre_execution_cb(tobj); 1072 if (err) { 1073 PRINT_FAIL("pre_execution_cb failed: %d\n", err); 1074 goto tobj_cleanup; 1075 } 1076 } 1077 1078 do_prog_test_run(bpf_program__fd(tprog), &retval, 1079 bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false); 1080 if (retval != subspec->retval && subspec->retval != POINTER_VALUE) { 1081 PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval); 1082 goto tobj_cleanup; 1083 } 1084 /* redo bpf_map__attach_struct_ops for each test */ 1085 while (links_cnt > 0) 1086 bpf_link__destroy(links[--links_cnt]); 1087 } 1088 1089 tobj_cleanup: 1090 while (links_cnt > 0) 1091 bpf_link__destroy(links[--links_cnt]); 1092 bpf_object__close(tobj); 1093 subtest_cleanup: 1094 test__end_subtest(); 1095 restore_capabilities(&caps); 1096 } 1097 1098 static void process_subtest(struct test_loader *tester, 1099 const char *skel_name, 1100 skel_elf_bytes_fn elf_bytes_factory) 1101 { 1102 LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name); 1103 struct test_spec *specs = NULL; 1104 struct bpf_object *obj = NULL; 1105 struct bpf_program *prog; 1106 const void *obj_bytes; 1107 int err, i, nr_progs; 1108 size_t obj_byte_cnt; 1109 1110 if (tester_init(tester) < 0) 1111 return; /* failed to initialize tester */ 1112 1113 obj_bytes = elf_bytes_factory(&obj_byte_cnt); 1114 obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts); 1115 if (!ASSERT_OK_PTR(obj, "obj_open_mem")) 1116 return; 1117 1118 nr_progs = 0; 1119 bpf_object__for_each_program(prog, obj) 1120 ++nr_progs; 1121 1122 specs = calloc(nr_progs, sizeof(struct test_spec)); 1123 if (!ASSERT_OK_PTR(specs, "specs_alloc")) 1124 return; 1125 1126 i = 0; 1127 bpf_object__for_each_program(prog, obj) { 1128 /* ignore tests for which we can't derive test specification */ 1129 err = parse_test_spec(tester, obj, prog, &specs[i++]); 1130 if (err) 1131 PRINT_FAIL("Can't parse test spec for program '%s'\n", 1132 bpf_program__name(prog)); 1133 } 1134 1135 i = 0; 1136 bpf_object__for_each_program(prog, obj) { 1137 struct test_spec *spec = &specs[i++]; 1138 1139 if (!spec->valid || spec->auxiliary) 1140 continue; 1141 1142 if (spec->mode_mask & PRIV) 1143 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt, 1144 specs, spec, false); 1145 if (spec->mode_mask & UNPRIV) 1146 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt, 1147 specs, spec, true); 1148 1149 } 1150 1151 for (i = 0; i < nr_progs; ++i) 1152 free_test_spec(&specs[i]); 1153 free(specs); 1154 bpf_object__close(obj); 1155 } 1156 1157 void test_loader__run_subtests(struct test_loader *tester, 1158 const char *skel_name, 1159 skel_elf_bytes_fn elf_bytes_factory) 1160 { 1161 /* see comment in run_subtest() for why we do this function nesting */ 1162 process_subtest(tester, skel_name, elf_bytes_factory); 1163 } 1164