1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/capability.h> 4 #include <stdlib.h> 5 #include <regex.h> 6 #include <test_progs.h> 7 #include <bpf/btf.h> 8 9 #include "autoconf_helper.h" 10 #include "disasm_helpers.h" 11 #include "unpriv_helpers.h" 12 #include "cap_helpers.h" 13 #include "jit_disasm_helpers.h" 14 15 #define str_has_pfx(str, pfx) \ 16 (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0) 17 18 #define TEST_LOADER_LOG_BUF_SZ 2097152 19 20 #define TEST_TAG_EXPECT_FAILURE "comment:test_expect_failure" 21 #define TEST_TAG_EXPECT_SUCCESS "comment:test_expect_success" 22 #define TEST_TAG_EXPECT_MSG_PFX "comment:test_expect_msg=" 23 #define TEST_TAG_EXPECT_XLATED_PFX "comment:test_expect_xlated=" 24 #define TEST_TAG_EXPECT_FAILURE_UNPRIV "comment:test_expect_failure_unpriv" 25 #define TEST_TAG_EXPECT_SUCCESS_UNPRIV "comment:test_expect_success_unpriv" 26 #define TEST_TAG_EXPECT_MSG_PFX_UNPRIV "comment:test_expect_msg_unpriv=" 27 #define TEST_TAG_EXPECT_XLATED_PFX_UNPRIV "comment:test_expect_xlated_unpriv=" 28 #define TEST_TAG_LOG_LEVEL_PFX "comment:test_log_level=" 29 #define TEST_TAG_PROG_FLAGS_PFX "comment:test_prog_flags=" 30 #define TEST_TAG_DESCRIPTION_PFX "comment:test_description=" 31 #define TEST_TAG_RETVAL_PFX "comment:test_retval=" 32 #define TEST_TAG_RETVAL_PFX_UNPRIV "comment:test_retval_unpriv=" 33 #define TEST_TAG_AUXILIARY "comment:test_auxiliary" 34 #define TEST_TAG_AUXILIARY_UNPRIV "comment:test_auxiliary_unpriv" 35 #define TEST_BTF_PATH "comment:test_btf_path=" 36 #define TEST_TAG_ARCH "comment:test_arch=" 37 #define TEST_TAG_JITED_PFX "comment:test_jited=" 38 #define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv=" 39 #define TEST_TAG_CAPS_UNPRIV "comment:test_caps_unpriv=" 40 #define TEST_TAG_LOAD_MODE_PFX "comment:load_mode=" 41 42 /* Warning: duplicated in bpf_misc.h */ 43 #define POINTER_VALUE 0xbadcafe 44 #define TEST_DATA_LEN 64 45 46 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 47 #define EFFICIENT_UNALIGNED_ACCESS 1 48 #else 49 #define EFFICIENT_UNALIGNED_ACCESS 0 50 #endif 51 52 static int sysctl_unpriv_disabled = -1; 53 54 enum mode { 55 PRIV = 1, 56 UNPRIV = 2 57 }; 58 59 enum load_mode { 60 JITED = 1 << 0, 61 NO_JITED = 1 << 1, 62 }; 63 64 struct expect_msg { 65 const char *substr; /* substring match */ 66 regex_t regex; 67 bool is_regex; 68 bool on_next_line; 69 }; 70 71 struct expected_msgs { 72 struct expect_msg *patterns; 73 size_t cnt; 74 }; 75 76 struct test_subspec { 77 char *name; 78 bool expect_failure; 79 struct expected_msgs expect_msgs; 80 struct expected_msgs expect_xlated; 81 struct expected_msgs jited; 82 int retval; 83 bool execute; 84 __u64 caps; 85 }; 86 87 struct test_spec { 88 const char *prog_name; 89 struct test_subspec priv; 90 struct test_subspec unpriv; 91 const char *btf_custom_path; 92 int log_level; 93 int prog_flags; 94 int mode_mask; 95 int arch_mask; 96 int load_mask; 97 bool auxiliary; 98 bool valid; 99 }; 100 101 static int tester_init(struct test_loader *tester) 102 { 103 if (!tester->log_buf) { 104 tester->log_buf_sz = TEST_LOADER_LOG_BUF_SZ; 105 tester->log_buf = calloc(tester->log_buf_sz, 1); 106 if (!ASSERT_OK_PTR(tester->log_buf, "tester_log_buf")) 107 return -ENOMEM; 108 } 109 110 return 0; 111 } 112 113 void test_loader_fini(struct test_loader *tester) 114 { 115 if (!tester) 116 return; 117 118 free(tester->log_buf); 119 } 120 121 static void free_msgs(struct expected_msgs *msgs) 122 { 123 int i; 124 125 for (i = 0; i < msgs->cnt; i++) 126 if (msgs->patterns[i].is_regex) 127 regfree(&msgs->patterns[i].regex); 128 free(msgs->patterns); 129 msgs->patterns = NULL; 130 msgs->cnt = 0; 131 } 132 133 static void free_test_spec(struct test_spec *spec) 134 { 135 /* Deallocate expect_msgs arrays. */ 136 free_msgs(&spec->priv.expect_msgs); 137 free_msgs(&spec->unpriv.expect_msgs); 138 free_msgs(&spec->priv.expect_xlated); 139 free_msgs(&spec->unpriv.expect_xlated); 140 free_msgs(&spec->priv.jited); 141 free_msgs(&spec->unpriv.jited); 142 143 free(spec->priv.name); 144 free(spec->unpriv.name); 145 spec->priv.name = NULL; 146 spec->unpriv.name = NULL; 147 } 148 149 /* Compiles regular expression matching pattern. 150 * Pattern has a special syntax: 151 * 152 * pattern := (<verbatim text> | regex)* 153 * regex := "{{" <posix extended regular expression> "}}" 154 * 155 * In other words, pattern is a verbatim text with inclusion 156 * of regular expressions enclosed in "{{" "}}" pairs. 157 * For example, pattern "foo{{[0-9]+}}" matches strings like 158 * "foo0", "foo007", etc. 159 */ 160 static int compile_regex(const char *pattern, regex_t *regex) 161 { 162 char err_buf[256], buf[256] = {}, *ptr, *buf_end; 163 const char *original_pattern = pattern; 164 bool in_regex = false; 165 int err; 166 167 buf_end = buf + sizeof(buf); 168 ptr = buf; 169 while (*pattern && ptr < buf_end - 2) { 170 if (!in_regex && str_has_pfx(pattern, "{{")) { 171 in_regex = true; 172 pattern += 2; 173 continue; 174 } 175 if (in_regex && str_has_pfx(pattern, "}}")) { 176 in_regex = false; 177 pattern += 2; 178 continue; 179 } 180 if (in_regex) { 181 *ptr++ = *pattern++; 182 continue; 183 } 184 /* list of characters that need escaping for extended posix regex */ 185 if (strchr(".[]\\()*+?{}|^$", *pattern)) { 186 *ptr++ = '\\'; 187 *ptr++ = *pattern++; 188 continue; 189 } 190 *ptr++ = *pattern++; 191 } 192 if (*pattern) { 193 PRINT_FAIL("Regexp too long: '%s'\n", original_pattern); 194 return -EINVAL; 195 } 196 if (in_regex) { 197 PRINT_FAIL("Regexp has open '{{' but no closing '}}': '%s'\n", original_pattern); 198 return -EINVAL; 199 } 200 err = regcomp(regex, buf, REG_EXTENDED | REG_NEWLINE); 201 if (err != 0) { 202 regerror(err, regex, err_buf, sizeof(err_buf)); 203 PRINT_FAIL("Regexp compilation error in '%s': '%s'\n", buf, err_buf); 204 return -EINVAL; 205 } 206 return 0; 207 } 208 209 static int __push_msg(const char *pattern, bool on_next_line, struct expected_msgs *msgs) 210 { 211 struct expect_msg *msg; 212 void *tmp; 213 int err; 214 215 tmp = realloc(msgs->patterns, 216 (1 + msgs->cnt) * sizeof(struct expect_msg)); 217 if (!tmp) { 218 ASSERT_FAIL("failed to realloc memory for messages\n"); 219 return -ENOMEM; 220 } 221 msgs->patterns = tmp; 222 msg = &msgs->patterns[msgs->cnt]; 223 msg->on_next_line = on_next_line; 224 msg->substr = pattern; 225 msg->is_regex = false; 226 if (strstr(pattern, "{{")) { 227 err = compile_regex(pattern, &msg->regex); 228 if (err) 229 return err; 230 msg->is_regex = true; 231 } 232 msgs->cnt += 1; 233 return 0; 234 } 235 236 static int clone_msgs(struct expected_msgs *from, struct expected_msgs *to) 237 { 238 struct expect_msg *msg; 239 int i, err; 240 241 for (i = 0; i < from->cnt; i++) { 242 msg = &from->patterns[i]; 243 err = __push_msg(msg->substr, msg->on_next_line, to); 244 if (err) 245 return err; 246 } 247 return 0; 248 } 249 250 static int push_msg(const char *substr, struct expected_msgs *msgs) 251 { 252 return __push_msg(substr, false, msgs); 253 } 254 255 static int push_disasm_msg(const char *regex_str, bool *on_next_line, struct expected_msgs *msgs) 256 { 257 int err; 258 259 if (strcmp(regex_str, "...") == 0) { 260 *on_next_line = false; 261 return 0; 262 } 263 err = __push_msg(regex_str, *on_next_line, msgs); 264 if (err) 265 return err; 266 *on_next_line = true; 267 return 0; 268 } 269 270 static int parse_int(const char *str, int *val, const char *name) 271 { 272 char *end; 273 long tmp; 274 275 errno = 0; 276 if (str_has_pfx(str, "0x")) 277 tmp = strtol(str + 2, &end, 16); 278 else 279 tmp = strtol(str, &end, 10); 280 if (errno || end[0] != '\0') { 281 PRINT_FAIL("failed to parse %s from '%s'\n", name, str); 282 return -EINVAL; 283 } 284 *val = tmp; 285 return 0; 286 } 287 288 static int parse_caps(const char *str, __u64 *val, const char *name) 289 { 290 int cap_flag = 0; 291 char *token = NULL, *saveptr = NULL; 292 293 char *str_cpy = strdup(str); 294 if (str_cpy == NULL) { 295 PRINT_FAIL("Memory allocation failed\n"); 296 return -EINVAL; 297 } 298 299 token = strtok_r(str_cpy, "|", &saveptr); 300 while (token != NULL) { 301 errno = 0; 302 if (!strncmp("CAP_", token, sizeof("CAP_") - 1)) { 303 PRINT_FAIL("define %s constant in bpf_misc.h, failed to parse caps\n", token); 304 return -EINVAL; 305 } 306 cap_flag = strtol(token, NULL, 10); 307 if (!cap_flag || errno) { 308 PRINT_FAIL("failed to parse caps %s\n", name); 309 return -EINVAL; 310 } 311 *val |= (1ULL << cap_flag); 312 token = strtok_r(NULL, "|", &saveptr); 313 } 314 315 free(str_cpy); 316 return 0; 317 } 318 319 static int parse_retval(const char *str, int *val, const char *name) 320 { 321 /* 322 * INT_MIN is defined as (-INT_MAX -1), i.e. it doesn't expand to a 323 * single int and cannot be parsed with strtol, so we handle it 324 * separately here. In addition, it expands to different expressions in 325 * different compilers so we use a prefixed _INT_MIN instead. 326 */ 327 if (strcmp(str, "_INT_MIN") == 0) { 328 *val = INT_MIN; 329 return 0; 330 } 331 332 return parse_int(str, val, name); 333 } 334 335 static void update_flags(int *flags, int flag, bool clear) 336 { 337 if (clear) 338 *flags &= ~flag; 339 else 340 *flags |= flag; 341 } 342 343 /* Matches a string of form '<pfx>[^=]=.*' and returns it's suffix. 344 * Used to parse btf_decl_tag values. 345 * Such values require unique prefix because compiler does not add 346 * same __attribute__((btf_decl_tag(...))) twice. 347 * Test suite uses two-component tags for such cases: 348 * 349 * <pfx> __COUNTER__ '=' 350 * 351 * For example, two consecutive __msg tags '__msg("foo") __msg("foo")' 352 * would be encoded as: 353 * 354 * [18] DECL_TAG 'comment:test_expect_msg=0=foo' type_id=15 component_idx=-1 355 * [19] DECL_TAG 'comment:test_expect_msg=1=foo' type_id=15 component_idx=-1 356 * 357 * And the purpose of this function is to extract 'foo' from the above. 358 */ 359 static const char *skip_dynamic_pfx(const char *s, const char *pfx) 360 { 361 const char *msg; 362 363 if (strncmp(s, pfx, strlen(pfx)) != 0) 364 return NULL; 365 msg = s + strlen(pfx); 366 msg = strchr(msg, '='); 367 if (!msg) 368 return NULL; 369 return msg + 1; 370 } 371 372 enum arch { 373 ARCH_UNKNOWN = 0x1, 374 ARCH_X86_64 = 0x2, 375 ARCH_ARM64 = 0x4, 376 ARCH_RISCV64 = 0x8, 377 ARCH_S390X = 0x10, 378 }; 379 380 static int get_current_arch(void) 381 { 382 #if defined(__x86_64__) 383 return ARCH_X86_64; 384 #elif defined(__aarch64__) 385 return ARCH_ARM64; 386 #elif defined(__riscv) && __riscv_xlen == 64 387 return ARCH_RISCV64; 388 #elif defined(__s390x__) 389 return ARCH_S390X; 390 #endif 391 return ARCH_UNKNOWN; 392 } 393 394 /* Uses btf_decl_tag attributes to describe the expected test 395 * behavior, see bpf_misc.h for detailed description of each attribute 396 * and attribute combinations. 397 */ 398 static int parse_test_spec(struct test_loader *tester, 399 struct bpf_object *obj, 400 struct bpf_program *prog, 401 struct test_spec *spec) 402 { 403 const char *description = NULL; 404 bool has_unpriv_result = false; 405 bool has_unpriv_retval = false; 406 bool unpriv_xlated_on_next_line = true; 407 bool xlated_on_next_line = true; 408 bool unpriv_jit_on_next_line; 409 bool jit_on_next_line; 410 bool collect_jit = false; 411 int func_id, i, err = 0; 412 u32 arch_mask = 0; 413 u32 load_mask = 0; 414 struct btf *btf; 415 enum arch arch; 416 417 memset(spec, 0, sizeof(*spec)); 418 419 spec->prog_name = bpf_program__name(prog); 420 spec->prog_flags = testing_prog_flags(); 421 422 btf = bpf_object__btf(obj); 423 if (!btf) { 424 ASSERT_FAIL("BPF object has no BTF"); 425 return -EINVAL; 426 } 427 428 func_id = btf__find_by_name_kind(btf, spec->prog_name, BTF_KIND_FUNC); 429 if (func_id < 0) { 430 ASSERT_FAIL("failed to find FUNC BTF type for '%s'", spec->prog_name); 431 return -EINVAL; 432 } 433 434 for (i = 1; i < btf__type_cnt(btf); i++) { 435 const char *s, *val, *msg; 436 const struct btf_type *t; 437 bool clear; 438 int flags; 439 440 t = btf__type_by_id(btf, i); 441 if (!btf_is_decl_tag(t)) 442 continue; 443 444 if (t->type != func_id || btf_decl_tag(t)->component_idx != -1) 445 continue; 446 447 s = btf__str_by_offset(btf, t->name_off); 448 if (str_has_pfx(s, TEST_TAG_DESCRIPTION_PFX)) { 449 description = s + sizeof(TEST_TAG_DESCRIPTION_PFX) - 1; 450 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE) == 0) { 451 spec->priv.expect_failure = true; 452 spec->mode_mask |= PRIV; 453 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS) == 0) { 454 spec->priv.expect_failure = false; 455 spec->mode_mask |= PRIV; 456 } else if (strcmp(s, TEST_TAG_EXPECT_FAILURE_UNPRIV) == 0) { 457 spec->unpriv.expect_failure = true; 458 spec->mode_mask |= UNPRIV; 459 has_unpriv_result = true; 460 } else if (strcmp(s, TEST_TAG_EXPECT_SUCCESS_UNPRIV) == 0) { 461 spec->unpriv.expect_failure = false; 462 spec->mode_mask |= UNPRIV; 463 has_unpriv_result = true; 464 } else if (strcmp(s, TEST_TAG_AUXILIARY) == 0) { 465 spec->auxiliary = true; 466 spec->mode_mask |= PRIV; 467 } else if (strcmp(s, TEST_TAG_AUXILIARY_UNPRIV) == 0) { 468 spec->auxiliary = true; 469 spec->mode_mask |= UNPRIV; 470 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX))) { 471 err = push_msg(msg, &spec->priv.expect_msgs); 472 if (err) 473 goto cleanup; 474 spec->mode_mask |= PRIV; 475 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_MSG_PFX_UNPRIV))) { 476 err = push_msg(msg, &spec->unpriv.expect_msgs); 477 if (err) 478 goto cleanup; 479 spec->mode_mask |= UNPRIV; 480 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX))) { 481 if (arch_mask == 0) { 482 PRINT_FAIL("__jited used before __arch_*"); 483 goto cleanup; 484 } 485 if (collect_jit) { 486 err = push_disasm_msg(msg, &jit_on_next_line, 487 &spec->priv.jited); 488 if (err) 489 goto cleanup; 490 spec->mode_mask |= PRIV; 491 } 492 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_JITED_PFX_UNPRIV))) { 493 if (arch_mask == 0) { 494 PRINT_FAIL("__unpriv_jited used before __arch_*"); 495 goto cleanup; 496 } 497 if (collect_jit) { 498 err = push_disasm_msg(msg, &unpriv_jit_on_next_line, 499 &spec->unpriv.jited); 500 if (err) 501 goto cleanup; 502 spec->mode_mask |= UNPRIV; 503 } 504 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX))) { 505 err = push_disasm_msg(msg, &xlated_on_next_line, 506 &spec->priv.expect_xlated); 507 if (err) 508 goto cleanup; 509 spec->mode_mask |= PRIV; 510 } else if ((msg = skip_dynamic_pfx(s, TEST_TAG_EXPECT_XLATED_PFX_UNPRIV))) { 511 err = push_disasm_msg(msg, &unpriv_xlated_on_next_line, 512 &spec->unpriv.expect_xlated); 513 if (err) 514 goto cleanup; 515 spec->mode_mask |= UNPRIV; 516 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX)) { 517 val = s + sizeof(TEST_TAG_RETVAL_PFX) - 1; 518 err = parse_retval(val, &spec->priv.retval, "__retval"); 519 if (err) 520 goto cleanup; 521 spec->priv.execute = true; 522 spec->mode_mask |= PRIV; 523 } else if (str_has_pfx(s, TEST_TAG_RETVAL_PFX_UNPRIV)) { 524 val = s + sizeof(TEST_TAG_RETVAL_PFX_UNPRIV) - 1; 525 err = parse_retval(val, &spec->unpriv.retval, "__retval_unpriv"); 526 if (err) 527 goto cleanup; 528 spec->mode_mask |= UNPRIV; 529 spec->unpriv.execute = true; 530 has_unpriv_retval = true; 531 } else if (str_has_pfx(s, TEST_TAG_LOG_LEVEL_PFX)) { 532 val = s + sizeof(TEST_TAG_LOG_LEVEL_PFX) - 1; 533 err = parse_int(val, &spec->log_level, "test log level"); 534 if (err) 535 goto cleanup; 536 } else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) { 537 val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1; 538 539 clear = val[0] == '!'; 540 if (clear) 541 val++; 542 543 if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) { 544 update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear); 545 } else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) { 546 update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear); 547 } else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) { 548 update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear); 549 } else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) { 550 update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear); 551 } else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) { 552 update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear); 553 } else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) { 554 update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); 555 } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { 556 update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); 557 } else /* assume numeric value */ { 558 err = parse_int(val, &flags, "test prog flags"); 559 if (err) 560 goto cleanup; 561 update_flags(&spec->prog_flags, flags, clear); 562 } 563 } else if (str_has_pfx(s, TEST_TAG_ARCH)) { 564 val = s + sizeof(TEST_TAG_ARCH) - 1; 565 if (strcmp(val, "X86_64") == 0) { 566 arch = ARCH_X86_64; 567 } else if (strcmp(val, "ARM64") == 0) { 568 arch = ARCH_ARM64; 569 } else if (strcmp(val, "RISCV64") == 0) { 570 arch = ARCH_RISCV64; 571 } else if (strcmp(val, "s390x") == 0) { 572 arch = ARCH_S390X; 573 } else { 574 PRINT_FAIL("bad arch spec: '%s'\n", val); 575 err = -EINVAL; 576 goto cleanup; 577 } 578 arch_mask |= arch; 579 collect_jit = get_current_arch() == arch; 580 unpriv_jit_on_next_line = true; 581 jit_on_next_line = true; 582 } else if (str_has_pfx(s, TEST_BTF_PATH)) { 583 spec->btf_custom_path = s + sizeof(TEST_BTF_PATH) - 1; 584 } else if (str_has_pfx(s, TEST_TAG_CAPS_UNPRIV)) { 585 val = s + sizeof(TEST_TAG_CAPS_UNPRIV) - 1; 586 err = parse_caps(val, &spec->unpriv.caps, "test caps"); 587 if (err) 588 goto cleanup; 589 spec->mode_mask |= UNPRIV; 590 } else if (str_has_pfx(s, TEST_TAG_LOAD_MODE_PFX)) { 591 val = s + sizeof(TEST_TAG_LOAD_MODE_PFX) - 1; 592 if (strcmp(val, "jited") == 0) { 593 load_mask = JITED; 594 } else if (strcmp(val, "no_jited") == 0) { 595 load_mask = NO_JITED; 596 } else { 597 PRINT_FAIL("bad load spec: '%s'", val); 598 err = -EINVAL; 599 goto cleanup; 600 } 601 } 602 } 603 604 spec->arch_mask = arch_mask ?: -1; 605 spec->load_mask = load_mask ?: (JITED | NO_JITED); 606 607 if (spec->mode_mask == 0) 608 spec->mode_mask = PRIV; 609 610 if (!description) 611 description = spec->prog_name; 612 613 if (spec->mode_mask & PRIV) { 614 spec->priv.name = strdup(description); 615 if (!spec->priv.name) { 616 PRINT_FAIL("failed to allocate memory for priv.name\n"); 617 err = -ENOMEM; 618 goto cleanup; 619 } 620 } 621 622 if (spec->mode_mask & UNPRIV) { 623 int descr_len = strlen(description); 624 const char *suffix = " @unpriv"; 625 char *name; 626 627 name = malloc(descr_len + strlen(suffix) + 1); 628 if (!name) { 629 PRINT_FAIL("failed to allocate memory for unpriv.name\n"); 630 err = -ENOMEM; 631 goto cleanup; 632 } 633 634 strcpy(name, description); 635 strcpy(&name[descr_len], suffix); 636 spec->unpriv.name = name; 637 } 638 639 if (spec->mode_mask & (PRIV | UNPRIV)) { 640 if (!has_unpriv_result) 641 spec->unpriv.expect_failure = spec->priv.expect_failure; 642 643 if (!has_unpriv_retval) { 644 spec->unpriv.retval = spec->priv.retval; 645 spec->unpriv.execute = spec->priv.execute; 646 } 647 648 if (spec->unpriv.expect_msgs.cnt == 0) 649 clone_msgs(&spec->priv.expect_msgs, &spec->unpriv.expect_msgs); 650 if (spec->unpriv.expect_xlated.cnt == 0) 651 clone_msgs(&spec->priv.expect_xlated, &spec->unpriv.expect_xlated); 652 if (spec->unpriv.jited.cnt == 0) 653 clone_msgs(&spec->priv.jited, &spec->unpriv.jited); 654 } 655 656 spec->valid = true; 657 658 return 0; 659 660 cleanup: 661 free_test_spec(spec); 662 return err; 663 } 664 665 static void prepare_case(struct test_loader *tester, 666 struct test_spec *spec, 667 struct bpf_object *obj, 668 struct bpf_program *prog) 669 { 670 int min_log_level = 0, prog_flags; 671 672 if (env.verbosity > VERBOSE_NONE) 673 min_log_level = 1; 674 if (env.verbosity > VERBOSE_VERY) 675 min_log_level = 2; 676 677 bpf_program__set_log_buf(prog, tester->log_buf, tester->log_buf_sz); 678 679 /* Make sure we set at least minimal log level, unless test requires 680 * even higher level already. Make sure to preserve independent log 681 * level 4 (verifier stats), though. 682 */ 683 if ((spec->log_level & 3) < min_log_level) 684 bpf_program__set_log_level(prog, (spec->log_level & 4) | min_log_level); 685 else 686 bpf_program__set_log_level(prog, spec->log_level); 687 688 prog_flags = bpf_program__flags(prog); 689 bpf_program__set_flags(prog, prog_flags | spec->prog_flags); 690 691 tester->log_buf[0] = '\0'; 692 } 693 694 static void emit_verifier_log(const char *log_buf, bool force) 695 { 696 if (!force && env.verbosity == VERBOSE_NONE) 697 return; 698 fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log_buf); 699 } 700 701 static void emit_xlated(const char *xlated, bool force) 702 { 703 if (!force && env.verbosity == VERBOSE_NONE) 704 return; 705 fprintf(stdout, "XLATED:\n=============\n%s=============\n", xlated); 706 } 707 708 static void emit_jited(const char *jited, bool force) 709 { 710 if (!force && env.verbosity == VERBOSE_NONE) 711 return; 712 fprintf(stdout, "JITED:\n=============\n%s=============\n", jited); 713 } 714 715 static void validate_msgs(char *log_buf, struct expected_msgs *msgs, 716 void (*emit_fn)(const char *buf, bool force)) 717 { 718 const char *log = log_buf, *prev_match; 719 regmatch_t reg_match[1]; 720 int prev_match_line; 721 int match_line; 722 int i, j, err; 723 724 prev_match_line = -1; 725 match_line = 0; 726 prev_match = log; 727 for (i = 0; i < msgs->cnt; i++) { 728 struct expect_msg *msg = &msgs->patterns[i]; 729 const char *match = NULL, *pat_status; 730 bool wrong_line = false; 731 732 if (!msg->is_regex) { 733 match = strstr(log, msg->substr); 734 if (match) 735 log = match + strlen(msg->substr); 736 } else { 737 err = regexec(&msg->regex, log, 1, reg_match, 0); 738 if (err == 0) { 739 match = log + reg_match[0].rm_so; 740 log += reg_match[0].rm_eo; 741 } 742 } 743 744 if (match) { 745 for (; prev_match < match; ++prev_match) 746 if (*prev_match == '\n') 747 ++match_line; 748 wrong_line = msg->on_next_line && prev_match_line >= 0 && 749 prev_match_line + 1 != match_line; 750 } 751 752 if (!match || wrong_line) { 753 PRINT_FAIL("expect_msg\n"); 754 if (env.verbosity == VERBOSE_NONE) 755 emit_fn(log_buf, true /*force*/); 756 for (j = 0; j <= i; j++) { 757 msg = &msgs->patterns[j]; 758 if (j < i) 759 pat_status = "MATCHED "; 760 else if (wrong_line) 761 pat_status = "WRONG LINE"; 762 else 763 pat_status = "EXPECTED "; 764 msg = &msgs->patterns[j]; 765 fprintf(stderr, "%s %s: '%s'\n", 766 pat_status, 767 msg->is_regex ? " REGEX" : "SUBSTR", 768 msg->substr); 769 } 770 if (wrong_line) { 771 fprintf(stderr, 772 "expecting match at line %d, actual match is at line %d\n", 773 prev_match_line + 1, match_line); 774 } 775 break; 776 } 777 778 prev_match_line = match_line; 779 } 780 } 781 782 struct cap_state { 783 __u64 old_caps; 784 bool initialized; 785 }; 786 787 static int drop_capabilities(struct cap_state *caps) 788 { 789 const __u64 caps_to_drop = (1ULL << CAP_SYS_ADMIN | 1ULL << CAP_NET_ADMIN | 790 1ULL << CAP_PERFMON | 1ULL << CAP_BPF); 791 int err; 792 793 err = cap_disable_effective(caps_to_drop, &caps->old_caps); 794 if (err) { 795 PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(-err)); 796 return err; 797 } 798 799 caps->initialized = true; 800 return 0; 801 } 802 803 static int restore_capabilities(struct cap_state *caps) 804 { 805 int err; 806 807 if (!caps->initialized) 808 return 0; 809 810 err = cap_enable_effective(caps->old_caps, NULL); 811 if (err) 812 PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(-err)); 813 caps->initialized = false; 814 return err; 815 } 816 817 static bool can_execute_unpriv(struct test_loader *tester, struct test_spec *spec) 818 { 819 if (sysctl_unpriv_disabled < 0) 820 sysctl_unpriv_disabled = get_unpriv_disabled() ? 1 : 0; 821 if (sysctl_unpriv_disabled) 822 return false; 823 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) 824 return false; 825 return true; 826 } 827 828 static bool is_unpriv_capable_map(struct bpf_map *map) 829 { 830 enum bpf_map_type type; 831 __u32 flags; 832 833 type = bpf_map__type(map); 834 835 switch (type) { 836 case BPF_MAP_TYPE_HASH: 837 case BPF_MAP_TYPE_PERCPU_HASH: 838 case BPF_MAP_TYPE_HASH_OF_MAPS: 839 flags = bpf_map__map_flags(map); 840 return !(flags & BPF_F_ZERO_SEED); 841 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 842 case BPF_MAP_TYPE_ARRAY: 843 case BPF_MAP_TYPE_RINGBUF: 844 case BPF_MAP_TYPE_PROG_ARRAY: 845 case BPF_MAP_TYPE_CGROUP_ARRAY: 846 case BPF_MAP_TYPE_PERCPU_ARRAY: 847 case BPF_MAP_TYPE_USER_RINGBUF: 848 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 849 case BPF_MAP_TYPE_CGROUP_STORAGE: 850 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 851 return true; 852 default: 853 return false; 854 } 855 } 856 857 static int do_prog_test_run(int fd_prog, int *retval, bool empty_opts) 858 { 859 __u8 tmp_out[TEST_DATA_LEN << 2] = {}; 860 __u8 tmp_in[TEST_DATA_LEN] = {}; 861 int err, saved_errno; 862 LIBBPF_OPTS(bpf_test_run_opts, topts, 863 .data_in = tmp_in, 864 .data_size_in = sizeof(tmp_in), 865 .data_out = tmp_out, 866 .data_size_out = sizeof(tmp_out), 867 .repeat = 1, 868 ); 869 870 if (empty_opts) { 871 memset(&topts, 0, sizeof(struct bpf_test_run_opts)); 872 topts.sz = sizeof(struct bpf_test_run_opts); 873 } 874 err = bpf_prog_test_run_opts(fd_prog, &topts); 875 saved_errno = errno; 876 877 if (err) { 878 PRINT_FAIL("FAIL: Unexpected bpf_prog_test_run error: %d (%s) ", 879 saved_errno, strerror(saved_errno)); 880 return err; 881 } 882 883 ASSERT_OK(0, "bpf_prog_test_run"); 884 *retval = topts.retval; 885 886 return 0; 887 } 888 889 static bool should_do_test_run(struct test_spec *spec, struct test_subspec *subspec) 890 { 891 if (!subspec->execute) 892 return false; 893 894 if (subspec->expect_failure) 895 return false; 896 897 if ((spec->prog_flags & BPF_F_ANY_ALIGNMENT) && !EFFICIENT_UNALIGNED_ACCESS) { 898 if (env.verbosity != VERBOSE_NONE) 899 printf("alignment prevents execution\n"); 900 return false; 901 } 902 903 return true; 904 } 905 906 /* Get a disassembly of BPF program after verifier applies all rewrites */ 907 static int get_xlated_program_text(int prog_fd, char *text, size_t text_sz) 908 { 909 struct bpf_insn *insn_start = NULL, *insn, *insn_end; 910 __u32 insns_cnt = 0, i; 911 char buf[64]; 912 FILE *out = NULL; 913 int err; 914 915 err = get_xlated_program(prog_fd, &insn_start, &insns_cnt); 916 if (!ASSERT_OK(err, "get_xlated_program")) 917 goto out; 918 out = fmemopen(text, text_sz, "w"); 919 if (!ASSERT_OK_PTR(out, "open_memstream")) 920 goto out; 921 insn_end = insn_start + insns_cnt; 922 insn = insn_start; 923 while (insn < insn_end) { 924 i = insn - insn_start; 925 insn = disasm_insn(insn, buf, sizeof(buf)); 926 fprintf(out, "%d: %s\n", i, buf); 927 } 928 fflush(out); 929 930 out: 931 free(insn_start); 932 if (out) 933 fclose(out); 934 return err; 935 } 936 937 /* this function is forced noinline and has short generic name to look better 938 * in test_progs output (in case of a failure) 939 */ 940 static noinline 941 void run_subtest(struct test_loader *tester, 942 struct bpf_object_open_opts *open_opts, 943 const void *obj_bytes, 944 size_t obj_byte_cnt, 945 struct test_spec *specs, 946 struct test_spec *spec, 947 bool unpriv) 948 { 949 struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv; 950 int current_runtime = is_jit_enabled() ? JITED : NO_JITED; 951 struct bpf_program *tprog = NULL, *tprog_iter; 952 struct bpf_link *link, *links[32] = {}; 953 struct test_spec *spec_iter; 954 struct cap_state caps = {}; 955 struct bpf_object *tobj; 956 struct bpf_map *map; 957 int retval, err, i; 958 int links_cnt = 0; 959 bool should_load; 960 961 if (!test__start_subtest(subspec->name)) 962 return; 963 964 if ((get_current_arch() & spec->arch_mask) == 0) { 965 test__skip(); 966 return; 967 } 968 969 if ((current_runtime & spec->load_mask) == 0) { 970 test__skip(); 971 return; 972 } 973 974 if (unpriv) { 975 if (!can_execute_unpriv(tester, spec)) { 976 test__skip(); 977 test__end_subtest(); 978 return; 979 } 980 if (drop_capabilities(&caps)) { 981 test__end_subtest(); 982 return; 983 } 984 if (subspec->caps) { 985 err = cap_enable_effective(subspec->caps, NULL); 986 if (err) { 987 PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err)); 988 goto subtest_cleanup; 989 } 990 } 991 } 992 993 /* Implicitly reset to NULL if next test case doesn't specify */ 994 open_opts->btf_custom_path = spec->btf_custom_path; 995 996 tobj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, open_opts); 997 if (!ASSERT_OK_PTR(tobj, "obj_open_mem")) /* shouldn't happen */ 998 goto subtest_cleanup; 999 1000 i = 0; 1001 bpf_object__for_each_program(tprog_iter, tobj) { 1002 spec_iter = &specs[i++]; 1003 should_load = false; 1004 1005 if (spec_iter->valid) { 1006 if (strcmp(bpf_program__name(tprog_iter), spec->prog_name) == 0) { 1007 tprog = tprog_iter; 1008 should_load = true; 1009 } 1010 1011 if (spec_iter->auxiliary && 1012 spec_iter->mode_mask & (unpriv ? UNPRIV : PRIV)) 1013 should_load = true; 1014 } 1015 1016 bpf_program__set_autoload(tprog_iter, should_load); 1017 } 1018 1019 prepare_case(tester, spec, tobj, tprog); 1020 1021 /* By default bpf_object__load() automatically creates all 1022 * maps declared in the skeleton. Some map types are only 1023 * allowed in priv mode. Disable autoload for such maps in 1024 * unpriv mode. 1025 */ 1026 bpf_object__for_each_map(map, tobj) 1027 bpf_map__set_autocreate(map, !unpriv || is_unpriv_capable_map(map)); 1028 1029 err = bpf_object__load(tobj); 1030 if (subspec->expect_failure) { 1031 if (!ASSERT_ERR(err, "unexpected_load_success")) { 1032 emit_verifier_log(tester->log_buf, false /*force*/); 1033 goto tobj_cleanup; 1034 } 1035 } else { 1036 if (!ASSERT_OK(err, "unexpected_load_failure")) { 1037 emit_verifier_log(tester->log_buf, true /*force*/); 1038 goto tobj_cleanup; 1039 } 1040 } 1041 emit_verifier_log(tester->log_buf, false /*force*/); 1042 validate_msgs(tester->log_buf, &subspec->expect_msgs, emit_verifier_log); 1043 1044 /* Restore capabilities because the kernel will silently ignore requests 1045 * for program info (such as xlated program text) if we are not 1046 * bpf-capable. Also, for some reason test_verifier executes programs 1047 * with all capabilities restored. Do the same here. 1048 */ 1049 if (restore_capabilities(&caps)) 1050 goto tobj_cleanup; 1051 1052 if (subspec->expect_xlated.cnt) { 1053 err = get_xlated_program_text(bpf_program__fd(tprog), 1054 tester->log_buf, tester->log_buf_sz); 1055 if (err) 1056 goto tobj_cleanup; 1057 emit_xlated(tester->log_buf, false /*force*/); 1058 validate_msgs(tester->log_buf, &subspec->expect_xlated, emit_xlated); 1059 } 1060 1061 if (subspec->jited.cnt) { 1062 err = get_jited_program_text(bpf_program__fd(tprog), 1063 tester->log_buf, tester->log_buf_sz); 1064 if (err == -EOPNOTSUPP) { 1065 printf("%s:SKIP: jited programs disassembly is not supported,\n", __func__); 1066 printf("%s:SKIP: tests are built w/o LLVM development libs\n", __func__); 1067 test__skip(); 1068 goto tobj_cleanup; 1069 } 1070 if (!ASSERT_EQ(err, 0, "get_jited_program_text")) 1071 goto tobj_cleanup; 1072 emit_jited(tester->log_buf, false /*force*/); 1073 validate_msgs(tester->log_buf, &subspec->jited, emit_jited); 1074 } 1075 1076 if (should_do_test_run(spec, subspec)) { 1077 /* Do bpf_map__attach_struct_ops() for each struct_ops map. 1078 * This should trigger bpf_struct_ops->reg callback on kernel side. 1079 */ 1080 bpf_object__for_each_map(map, tobj) { 1081 if (!bpf_map__autocreate(map) || 1082 bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS) 1083 continue; 1084 if (links_cnt >= ARRAY_SIZE(links)) { 1085 PRINT_FAIL("too many struct_ops maps"); 1086 goto tobj_cleanup; 1087 } 1088 link = bpf_map__attach_struct_ops(map); 1089 if (!link) { 1090 PRINT_FAIL("bpf_map__attach_struct_ops failed for map %s: err=%d\n", 1091 bpf_map__name(map), -errno); 1092 goto tobj_cleanup; 1093 } 1094 links[links_cnt++] = link; 1095 } 1096 1097 if (tester->pre_execution_cb) { 1098 err = tester->pre_execution_cb(tobj); 1099 if (err) { 1100 PRINT_FAIL("pre_execution_cb failed: %d\n", err); 1101 goto tobj_cleanup; 1102 } 1103 } 1104 1105 err = do_prog_test_run(bpf_program__fd(tprog), &retval, 1106 bpf_program__type(tprog) == BPF_PROG_TYPE_SYSCALL ? true : false); 1107 if (!err && retval != subspec->retval && subspec->retval != POINTER_VALUE) { 1108 PRINT_FAIL("Unexpected retval: %d != %d\n", retval, subspec->retval); 1109 goto tobj_cleanup; 1110 } 1111 /* redo bpf_map__attach_struct_ops for each test */ 1112 while (links_cnt > 0) 1113 bpf_link__destroy(links[--links_cnt]); 1114 } 1115 1116 tobj_cleanup: 1117 while (links_cnt > 0) 1118 bpf_link__destroy(links[--links_cnt]); 1119 bpf_object__close(tobj); 1120 subtest_cleanup: 1121 test__end_subtest(); 1122 restore_capabilities(&caps); 1123 } 1124 1125 static void process_subtest(struct test_loader *tester, 1126 const char *skel_name, 1127 skel_elf_bytes_fn elf_bytes_factory) 1128 { 1129 LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = skel_name); 1130 struct test_spec *specs = NULL; 1131 struct bpf_object *obj = NULL; 1132 struct bpf_program *prog; 1133 const void *obj_bytes; 1134 int err, i, nr_progs; 1135 size_t obj_byte_cnt; 1136 1137 if (tester_init(tester) < 0) 1138 return; /* failed to initialize tester */ 1139 1140 obj_bytes = elf_bytes_factory(&obj_byte_cnt); 1141 obj = bpf_object__open_mem(obj_bytes, obj_byte_cnt, &open_opts); 1142 if (!ASSERT_OK_PTR(obj, "obj_open_mem")) 1143 return; 1144 1145 nr_progs = 0; 1146 bpf_object__for_each_program(prog, obj) 1147 ++nr_progs; 1148 1149 specs = calloc(nr_progs, sizeof(struct test_spec)); 1150 if (!ASSERT_OK_PTR(specs, "specs_alloc")) 1151 return; 1152 1153 i = 0; 1154 bpf_object__for_each_program(prog, obj) { 1155 /* ignore tests for which we can't derive test specification */ 1156 err = parse_test_spec(tester, obj, prog, &specs[i++]); 1157 if (err) 1158 PRINT_FAIL("Can't parse test spec for program '%s'\n", 1159 bpf_program__name(prog)); 1160 } 1161 1162 i = 0; 1163 bpf_object__for_each_program(prog, obj) { 1164 struct test_spec *spec = &specs[i++]; 1165 1166 if (!spec->valid || spec->auxiliary) 1167 continue; 1168 1169 if (spec->mode_mask & PRIV) 1170 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt, 1171 specs, spec, false); 1172 if (spec->mode_mask & UNPRIV) 1173 run_subtest(tester, &open_opts, obj_bytes, obj_byte_cnt, 1174 specs, spec, true); 1175 1176 } 1177 1178 for (i = 0; i < nr_progs; ++i) 1179 free_test_spec(&specs[i]); 1180 free(specs); 1181 bpf_object__close(obj); 1182 } 1183 1184 void test_loader__run_subtests(struct test_loader *tester, 1185 const char *skel_name, 1186 skel_elf_bytes_fn elf_bytes_factory) 1187 { 1188 /* see comment in run_subtest() for why we do this function nesting */ 1189 process_subtest(tester, skel_name, elf_bytes_factory); 1190 } 1191