1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (C) 2019 Netronome Systems, Inc. */ 3 /* Copyright (C) 2020 Facebook, Inc. */ 4 #include <ctype.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <errno.h> 8 #include <sys/mman.h> 9 #include <alloca.h> 10 #include <bpf/bpf.h> 11 #include <bpf/libbpf.h> 12 #include "disasm.h" 13 #include "test_progs.h" 14 #include "testing_helpers.h" 15 #include <linux/membarrier.h> 16 #include <linux/userfaultfd.h> 17 18 int parse_num_list(const char *s, bool **num_set, int *num_set_len) 19 { 20 int i, set_len = 0, new_len, num, start = 0, end = -1; 21 bool *set = NULL, *tmp, parsing_end = false; 22 char *next; 23 24 while (s[0]) { 25 errno = 0; 26 num = strtol(s, &next, 10); 27 if (errno) 28 return -errno; 29 30 if (parsing_end) 31 end = num; 32 else 33 start = num; 34 35 if (!parsing_end && *next == '-') { 36 s = next + 1; 37 parsing_end = true; 38 continue; 39 } else if (*next == ',') { 40 parsing_end = false; 41 s = next + 1; 42 end = num; 43 } else if (*next == '\0') { 44 parsing_end = false; 45 s = next; 46 end = num; 47 } else { 48 return -EINVAL; 49 } 50 51 if (start > end) 52 return -EINVAL; 53 54 if (end + 1 > set_len) { 55 new_len = end + 1; 56 tmp = realloc(set, new_len); 57 if (!tmp) { 58 free(set); 59 return -ENOMEM; 60 } 61 for (i = set_len; i < start; i++) 62 tmp[i] = false; 63 set = tmp; 64 set_len = new_len; 65 } 66 for (i = start; i <= end; i++) 67 set[i] = true; 68 } 69 70 if (!set || parsing_end) 71 return -EINVAL; 72 73 *num_set = set; 74 *num_set_len = set_len; 75 76 return 0; 77 } 78 79 static int do_insert_test(struct test_filter_set *set, 80 char *test_str, 81 char *subtest_str) 82 { 83 struct test_filter *tmp, *test; 84 char **ctmp; 85 int i; 86 87 for (i = 0; i < set->cnt; i++) { 88 test = &set->tests[i]; 89 90 if (strcmp(test_str, test->name) == 0) { 91 free(test_str); 92 goto subtest; 93 } 94 } 95 96 tmp = realloc(set->tests, sizeof(*test) * (set->cnt + 1)); 97 if (!tmp) 98 return -ENOMEM; 99 100 set->tests = tmp; 101 test = &set->tests[set->cnt]; 102 103 test->name = test_str; 104 test->subtests = NULL; 105 test->subtest_cnt = 0; 106 107 set->cnt++; 108 109 subtest: 110 if (!subtest_str) 111 return 0; 112 113 for (i = 0; i < test->subtest_cnt; i++) { 114 if (strcmp(subtest_str, test->subtests[i]) == 0) { 115 free(subtest_str); 116 return 0; 117 } 118 } 119 120 ctmp = realloc(test->subtests, 121 sizeof(*test->subtests) * (test->subtest_cnt + 1)); 122 if (!ctmp) 123 return -ENOMEM; 124 125 test->subtests = ctmp; 126 test->subtests[test->subtest_cnt] = subtest_str; 127 128 test->subtest_cnt++; 129 130 return 0; 131 } 132 133 static int insert_test(struct test_filter_set *set, 134 char *test_spec, 135 bool is_glob_pattern) 136 { 137 char *pattern, *subtest_str, *ext_test_str, *ext_subtest_str = NULL; 138 int glob_chars = 0; 139 140 if (is_glob_pattern) { 141 pattern = "%s"; 142 } else { 143 pattern = "*%s*"; 144 glob_chars = 2; 145 } 146 147 subtest_str = strchr(test_spec, '/'); 148 if (subtest_str) { 149 *subtest_str = '\0'; 150 subtest_str += 1; 151 } 152 153 ext_test_str = malloc(strlen(test_spec) + glob_chars + 1); 154 if (!ext_test_str) 155 goto err; 156 157 sprintf(ext_test_str, pattern, test_spec); 158 159 if (subtest_str) { 160 ext_subtest_str = malloc(strlen(subtest_str) + glob_chars + 1); 161 if (!ext_subtest_str) 162 goto err; 163 164 sprintf(ext_subtest_str, pattern, subtest_str); 165 } 166 167 return do_insert_test(set, ext_test_str, ext_subtest_str); 168 169 err: 170 free(ext_test_str); 171 free(ext_subtest_str); 172 173 return -ENOMEM; 174 } 175 176 int parse_test_list_file(const char *path, 177 struct test_filter_set *set, 178 bool is_glob_pattern) 179 { 180 char *buf = NULL, *capture_start, *capture_end, *scan_end; 181 size_t buflen = 0; 182 int err = 0; 183 FILE *f; 184 185 f = fopen(path, "r"); 186 if (!f) { 187 err = -errno; 188 fprintf(stderr, "Failed to open '%s': %d\n", path, err); 189 return err; 190 } 191 192 while (getline(&buf, &buflen, f) != -1) { 193 capture_start = buf; 194 195 while (isspace(*capture_start)) 196 ++capture_start; 197 198 capture_end = capture_start; 199 scan_end = capture_start; 200 201 while (*scan_end && *scan_end != '#') { 202 if (!isspace(*scan_end)) 203 capture_end = scan_end; 204 205 ++scan_end; 206 } 207 208 if (capture_end == capture_start) 209 continue; 210 211 *(++capture_end) = '\0'; 212 213 err = insert_test(set, capture_start, is_glob_pattern); 214 if (err) 215 break; 216 } 217 218 free(buf); 219 fclose(f); 220 return err; 221 } 222 223 int parse_test_list(const char *s, 224 struct test_filter_set *set, 225 bool is_glob_pattern) 226 { 227 char *input, *state = NULL, *test_spec; 228 int err = 0, cnt = 0; 229 230 input = strdup(s); 231 if (!input) 232 return -ENOMEM; 233 234 while ((test_spec = strtok_r(cnt++ ? NULL : input, ",", &state))) { 235 err = insert_test(set, test_spec, is_glob_pattern); 236 if (err) 237 break; 238 } 239 240 free(input); 241 return err; 242 } 243 244 __u32 link_info_prog_id(const struct bpf_link *link, struct bpf_link_info *info) 245 { 246 __u32 info_len = sizeof(*info); 247 int err; 248 249 memset(info, 0, sizeof(*info)); 250 err = bpf_link_get_info_by_fd(bpf_link__fd(link), info, &info_len); 251 if (err) { 252 printf("failed to get link info: %d\n", -errno); 253 return 0; 254 } 255 return info->prog_id; 256 } 257 258 int extra_prog_load_log_flags = 0; 259 260 int testing_prog_flags(void) 261 { 262 static int cached_flags = -1; 263 static int prog_flags[] = { BPF_F_TEST_RND_HI32, BPF_F_TEST_REG_INVARIANTS }; 264 static struct bpf_insn insns[] = { 265 BPF_MOV64_IMM(BPF_REG_0, 0), 266 BPF_EXIT_INSN(), 267 }; 268 int insn_cnt = ARRAY_SIZE(insns), i, fd, flags = 0; 269 LIBBPF_OPTS(bpf_prog_load_opts, opts); 270 271 if (cached_flags >= 0) 272 return cached_flags; 273 274 for (i = 0; i < ARRAY_SIZE(prog_flags); i++) { 275 opts.prog_flags = prog_flags[i]; 276 fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "flag-test", "GPL", 277 insns, insn_cnt, &opts); 278 if (fd >= 0) { 279 flags |= prog_flags[i]; 280 close(fd); 281 } 282 } 283 284 cached_flags = flags; 285 return cached_flags; 286 } 287 288 int bpf_prog_test_load(const char *file, enum bpf_prog_type type, 289 struct bpf_object **pobj, int *prog_fd) 290 { 291 LIBBPF_OPTS(bpf_object_open_opts, opts, 292 .kernel_log_level = extra_prog_load_log_flags, 293 ); 294 struct bpf_object *obj; 295 struct bpf_program *prog; 296 int err; 297 298 obj = bpf_object__open_file(file, &opts); 299 if (!obj) 300 return -errno; 301 302 prog = bpf_object__next_program(obj, NULL); 303 if (!prog) { 304 err = -ENOENT; 305 goto err_out; 306 } 307 308 if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type) 309 bpf_program__set_type(prog, type); 310 311 bpf_program__add_flags(prog, testing_prog_flags()); 312 313 err = bpf_object__load(obj); 314 if (err) 315 goto err_out; 316 317 *pobj = obj; 318 *prog_fd = bpf_program__fd(prog); 319 320 return 0; 321 err_out: 322 bpf_object__close(obj); 323 return err; 324 } 325 326 int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, 327 size_t insns_cnt, const char *license, 328 __u32 kern_version, char *log_buf, 329 size_t log_buf_sz) 330 { 331 LIBBPF_OPTS(bpf_prog_load_opts, opts, 332 .kern_version = kern_version, 333 .prog_flags = testing_prog_flags(), 334 .log_level = extra_prog_load_log_flags, 335 .log_buf = log_buf, 336 .log_size = log_buf_sz, 337 ); 338 339 return bpf_prog_load(type, NULL, license, insns, insns_cnt, &opts); 340 } 341 342 __u64 read_perf_max_sample_freq(void) 343 { 344 __u64 sample_freq = 5000; /* fallback to 5000 on error */ 345 FILE *f; 346 347 f = fopen("/proc/sys/kernel/perf_event_max_sample_rate", "r"); 348 if (f == NULL) { 349 printf("Failed to open /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 350 "return default value: 5000\n", -errno); 351 return sample_freq; 352 } 353 if (fscanf(f, "%llu", &sample_freq) != 1) { 354 printf("Failed to parse /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 355 "return default value: 5000\n", -errno); 356 } 357 358 fclose(f); 359 return sample_freq; 360 } 361 362 int finit_module(int fd, const char *param_values, int flags) 363 { 364 return syscall(__NR_finit_module, fd, param_values, flags); 365 } 366 367 int delete_module(const char *name, int flags) 368 { 369 return syscall(__NR_delete_module, name, flags); 370 } 371 372 int try_unload_module(const char *name, int retries, bool verbose) 373 { 374 int ret, cnt = 0; 375 376 if (kern_sync_rcu()) 377 fprintf(stdout, "Failed to trigger kernel-side RCU sync!\n"); 378 379 for (;;) { 380 ret = delete_module(name, 0); 381 if (!ret || errno != EAGAIN) 382 break; 383 if (++cnt > retries) { 384 fprintf(stdout, "Unload of %s timed out\n", name); 385 break; 386 } 387 usleep(100); 388 } 389 390 if (ret) { 391 if (errno == ENOENT) { 392 if (verbose) 393 fprintf(stdout, "%s.ko is already unloaded.\n", name); 394 return -1; 395 } 396 fprintf(stdout, "Failed to unload %s.ko from kernel: %d\n", name, -errno); 397 return -1; 398 } 399 if (verbose) 400 fprintf(stdout, "Successfully unloaded %s.ko.\n", name); 401 return 0; 402 } 403 404 int unload_module(const char *name, bool verbose) 405 { 406 return try_unload_module(name, 10000, verbose); 407 } 408 409 static int __load_module(const char *path, const char *param_values, bool verbose) 410 { 411 int fd; 412 413 if (verbose) 414 fprintf(stdout, "Loading %s...\n", path); 415 416 fd = open(path, O_RDONLY); 417 if (fd < 0) { 418 fprintf(stdout, "Can't find %s kernel module: %d\n", path, -errno); 419 return -ENOENT; 420 } 421 if (finit_module(fd, param_values, 0)) { 422 fprintf(stdout, "Failed to load %s into the kernel: %d\n", path, -errno); 423 close(fd); 424 return -EINVAL; 425 } 426 close(fd); 427 428 if (verbose) 429 fprintf(stdout, "Successfully loaded %s.\n", path); 430 return 0; 431 } 432 433 int load_module_params(const char *path, const char *param_values, bool verbose) 434 { 435 return __load_module(path, param_values, verbose); 436 } 437 438 int load_module(const char *path, bool verbose) 439 { 440 return __load_module(path, "", verbose); 441 } 442 443 int unload_bpf_testmod(bool verbose) 444 { 445 return unload_module("bpf_testmod", verbose); 446 } 447 448 int load_bpf_testmod(bool verbose) 449 { 450 return load_module("bpf_testmod.ko", verbose); 451 } 452 453 /* 454 * Trigger synchronize_rcu() in kernel. 455 */ 456 int kern_sync_rcu(void) 457 { 458 return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, 0, 0); 459 } 460 461 int uffd_block_page(void *fault_addr) 462 { 463 struct uffdio_register uffd_register = {}; 464 struct uffdio_api uffd_api = {}; 465 int uffd; 466 467 uffd = syscall(__NR_userfaultfd, O_CLOEXEC); 468 if (uffd < 0) 469 return -errno; 470 471 uffd_api.api = UFFD_API; 472 uffd_api.features = 0; 473 if (ioctl(uffd, UFFDIO_API, &uffd_api)) { 474 close(uffd); 475 return -1; 476 } 477 478 uffd_register.range.start = (unsigned long)fault_addr; 479 uffd_register.range.len = getpagesize(); 480 uffd_register.mode = UFFDIO_REGISTER_MODE_MISSING; 481 if (ioctl(uffd, UFFDIO_REGISTER, &uffd_register)) { 482 close(uffd); 483 return -1; 484 } 485 return uffd; 486 } 487 488 int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt) 489 { 490 __u32 buf_element_size = sizeof(struct bpf_insn); 491 struct bpf_prog_info info = {}; 492 __u32 info_len = sizeof(info); 493 __u32 xlated_prog_len; 494 495 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 496 perror("bpf_prog_get_info_by_fd failed"); 497 return -1; 498 } 499 500 xlated_prog_len = info.xlated_prog_len; 501 if (xlated_prog_len % buf_element_size) { 502 printf("Program length %u is not multiple of %u\n", 503 xlated_prog_len, buf_element_size); 504 return -1; 505 } 506 507 *cnt = xlated_prog_len / buf_element_size; 508 *buf = calloc(*cnt, buf_element_size); 509 if (!*buf) { 510 perror("can't allocate xlated program buffer"); 511 return -ENOMEM; 512 } 513 514 bzero(&info, sizeof(info)); 515 info.xlated_prog_len = xlated_prog_len; 516 info.xlated_prog_insns = (__u64)(unsigned long)*buf; 517 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 518 perror("second bpf_prog_get_info_by_fd failed"); 519 goto out_free_buf; 520 } 521 522 return 0; 523 524 out_free_buf: 525 free(*buf); 526 *buf = NULL; 527 return -1; 528 } 529 530 bool is_jit_enabled(void) 531 { 532 const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable"; 533 bool enabled = false; 534 int sysctl_fd; 535 536 sysctl_fd = open(jit_sysctl, O_RDONLY); 537 if (sysctl_fd != -1) { 538 char tmpc; 539 540 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1) 541 enabled = (tmpc != '0'); 542 close(sysctl_fd); 543 } 544 545 return enabled; 546 } 547 548 /* 549 * Whether the kernel accepts a program using more than 512 bytes of stack, 550 * which depends on the JIT in use. Probed once with a program that stores 551 * at the 2 KiB depth. Only the verifier's verdict on that store is cached: 552 * a load that fails for another reason, such as a missing capability, is 553 * reported and probed again on the next call. 554 */ 555 bool is_large_stack_supported(void) 556 { 557 static int supported = -1; 558 struct bpf_insn insns[] = { 559 BPF_ST_MEM(BPF_DW, BPF_REG_10, -2048, 0), 560 BPF_MOV64_IMM(BPF_REG_0, 0), 561 BPF_EXIT_INSN(), 562 }; 563 char log[1024] = {}; 564 LIBBPF_OPTS(bpf_prog_load_opts, opts, 565 .log_buf = log, 566 .log_size = sizeof(log), 567 .log_level = 1, 568 ); 569 int fd; 570 571 if (supported >= 0) 572 return supported; 573 574 fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, ARRAY_SIZE(insns), 575 &opts); 576 if (fd >= 0) { 577 close(fd); 578 supported = 1; 579 } else if (strstr(log, "invalid write to stack")) { 580 supported = 0; 581 } else { 582 fprintf(stderr, "%s: probe failed with errno %d, assuming 512 bytes:\n%s", 583 __func__, errno, log); 584 return false; 585 } 586 return supported; 587 } 588 589 int stack_mprotect(void) 590 { 591 void *buf; 592 long sz; 593 int ret; 594 595 sz = sysconf(_SC_PAGESIZE); 596 if (sz < 0) 597 return sz; 598 599 buf = alloca(sz * 3); 600 ret = mprotect((void *)(((unsigned long)(buf + sz)) & ~(sz - 1)), sz, 601 PROT_READ | PROT_WRITE | PROT_EXEC); 602 return ret; 603 } 604 605 int compare_text_to_expected(const char *actual, const char *expected) 606 { 607 char exp_path[] = "/tmp/selftest_expected.XXXXXX"; 608 char act_path[] = "/tmp/selftest_actual.XXXXXX"; 609 char buf[512], cmd[128]; 610 int exp_fd, act_fd; 611 FILE *p; 612 613 if (!strcmp(actual, expected)) 614 return 0; 615 616 exp_fd = mkstemp(exp_path); 617 act_fd = mkstemp(act_path); 618 if (exp_fd < 0 || act_fd < 0) { 619 fprintf(stdout, "output differs, no temp file for a diff\n"); 620 goto out; 621 } 622 623 dprintf(exp_fd, "%s", expected); 624 dprintf(act_fd, "%s", actual); 625 626 snprintf(cmd, sizeof(cmd), "diff -u '%s' '%s'", exp_path, act_path); 627 p = popen(cmd, "r"); 628 if (!p) { 629 fprintf(stdout, "output differs, '%s' did not run\n", cmd); 630 goto out; 631 } 632 while (fgets(buf, sizeof(buf), p)) 633 fputs(buf, stdout); 634 pclose(p); 635 636 out: 637 if (exp_fd >= 0) { 638 close(exp_fd); 639 unlink(exp_path); 640 } 641 if (act_fd >= 0) { 642 close(act_fd); 643 unlink(act_path); 644 } 645 return -1; 646 } 647