1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (C) 2019 Netronome Systems, Inc. */ 3 /* Copyright (C) 2020 Facebook, Inc. */ 4 #include <ctype.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <errno.h> 8 #include <sys/mman.h> 9 #include <alloca.h> 10 #include <bpf/bpf.h> 11 #include <bpf/libbpf.h> 12 #include "disasm.h" 13 #include "test_progs.h" 14 #include "testing_helpers.h" 15 #include <linux/membarrier.h> 16 #include <linux/userfaultfd.h> 17 18 int parse_num_list(const char *s, bool **num_set, int *num_set_len) 19 { 20 int i, set_len = 0, new_len, num, start = 0, end = -1; 21 bool *set = NULL, *tmp, parsing_end = false; 22 char *next; 23 24 while (s[0]) { 25 errno = 0; 26 num = strtol(s, &next, 10); 27 if (errno) 28 return -errno; 29 30 if (parsing_end) 31 end = num; 32 else 33 start = num; 34 35 if (!parsing_end && *next == '-') { 36 s = next + 1; 37 parsing_end = true; 38 continue; 39 } else if (*next == ',') { 40 parsing_end = false; 41 s = next + 1; 42 end = num; 43 } else if (*next == '\0') { 44 parsing_end = false; 45 s = next; 46 end = num; 47 } else { 48 return -EINVAL; 49 } 50 51 if (start > end) 52 return -EINVAL; 53 54 if (end + 1 > set_len) { 55 new_len = end + 1; 56 tmp = realloc(set, new_len); 57 if (!tmp) { 58 free(set); 59 return -ENOMEM; 60 } 61 for (i = set_len; i < start; i++) 62 tmp[i] = false; 63 set = tmp; 64 set_len = new_len; 65 } 66 for (i = start; i <= end; i++) 67 set[i] = true; 68 } 69 70 if (!set || parsing_end) 71 return -EINVAL; 72 73 *num_set = set; 74 *num_set_len = set_len; 75 76 return 0; 77 } 78 79 static int do_insert_test(struct test_filter_set *set, 80 char *test_str, 81 char *subtest_str) 82 { 83 struct test_filter *tmp, *test; 84 char **ctmp; 85 int i; 86 87 for (i = 0; i < set->cnt; i++) { 88 test = &set->tests[i]; 89 90 if (strcmp(test_str, test->name) == 0) { 91 free(test_str); 92 goto subtest; 93 } 94 } 95 96 tmp = realloc(set->tests, sizeof(*test) * (set->cnt + 1)); 97 if (!tmp) 98 return -ENOMEM; 99 100 set->tests = tmp; 101 test = &set->tests[set->cnt]; 102 103 test->name = test_str; 104 test->subtests = NULL; 105 test->subtest_cnt = 0; 106 107 set->cnt++; 108 109 subtest: 110 if (!subtest_str) 111 return 0; 112 113 for (i = 0; i < test->subtest_cnt; i++) { 114 if (strcmp(subtest_str, test->subtests[i]) == 0) { 115 free(subtest_str); 116 return 0; 117 } 118 } 119 120 ctmp = realloc(test->subtests, 121 sizeof(*test->subtests) * (test->subtest_cnt + 1)); 122 if (!ctmp) 123 return -ENOMEM; 124 125 test->subtests = ctmp; 126 test->subtests[test->subtest_cnt] = subtest_str; 127 128 test->subtest_cnt++; 129 130 return 0; 131 } 132 133 static int insert_test(struct test_filter_set *set, 134 char *test_spec, 135 bool is_glob_pattern) 136 { 137 char *pattern, *subtest_str, *ext_test_str, *ext_subtest_str = NULL; 138 int glob_chars = 0; 139 140 if (is_glob_pattern) { 141 pattern = "%s"; 142 } else { 143 pattern = "*%s*"; 144 glob_chars = 2; 145 } 146 147 subtest_str = strchr(test_spec, '/'); 148 if (subtest_str) { 149 *subtest_str = '\0'; 150 subtest_str += 1; 151 } 152 153 ext_test_str = malloc(strlen(test_spec) + glob_chars + 1); 154 if (!ext_test_str) 155 goto err; 156 157 sprintf(ext_test_str, pattern, test_spec); 158 159 if (subtest_str) { 160 ext_subtest_str = malloc(strlen(subtest_str) + glob_chars + 1); 161 if (!ext_subtest_str) 162 goto err; 163 164 sprintf(ext_subtest_str, pattern, subtest_str); 165 } 166 167 return do_insert_test(set, ext_test_str, ext_subtest_str); 168 169 err: 170 free(ext_test_str); 171 free(ext_subtest_str); 172 173 return -ENOMEM; 174 } 175 176 int parse_test_list_file(const char *path, 177 struct test_filter_set *set, 178 bool is_glob_pattern) 179 { 180 char *buf = NULL, *capture_start, *capture_end, *scan_end; 181 size_t buflen = 0; 182 int err = 0; 183 FILE *f; 184 185 f = fopen(path, "r"); 186 if (!f) { 187 err = -errno; 188 fprintf(stderr, "Failed to open '%s': %d\n", path, err); 189 return err; 190 } 191 192 while (getline(&buf, &buflen, f) != -1) { 193 capture_start = buf; 194 195 while (isspace(*capture_start)) 196 ++capture_start; 197 198 capture_end = capture_start; 199 scan_end = capture_start; 200 201 while (*scan_end && *scan_end != '#') { 202 if (!isspace(*scan_end)) 203 capture_end = scan_end; 204 205 ++scan_end; 206 } 207 208 if (capture_end == capture_start) 209 continue; 210 211 *(++capture_end) = '\0'; 212 213 err = insert_test(set, capture_start, is_glob_pattern); 214 if (err) 215 break; 216 } 217 218 free(buf); 219 fclose(f); 220 return err; 221 } 222 223 int parse_test_list(const char *s, 224 struct test_filter_set *set, 225 bool is_glob_pattern) 226 { 227 char *input, *state = NULL, *test_spec; 228 int err = 0, cnt = 0; 229 230 input = strdup(s); 231 if (!input) 232 return -ENOMEM; 233 234 while ((test_spec = strtok_r(cnt++ ? NULL : input, ",", &state))) { 235 err = insert_test(set, test_spec, is_glob_pattern); 236 if (err) 237 break; 238 } 239 240 free(input); 241 return err; 242 } 243 244 __u32 link_info_prog_id(const struct bpf_link *link, struct bpf_link_info *info) 245 { 246 __u32 info_len = sizeof(*info); 247 int err; 248 249 memset(info, 0, sizeof(*info)); 250 err = bpf_link_get_info_by_fd(bpf_link__fd(link), info, &info_len); 251 if (err) { 252 printf("failed to get link info: %d\n", -errno); 253 return 0; 254 } 255 return info->prog_id; 256 } 257 258 int extra_prog_load_log_flags = 0; 259 260 int testing_prog_flags(void) 261 { 262 static int cached_flags = -1; 263 static int prog_flags[] = { BPF_F_TEST_RND_HI32, BPF_F_TEST_REG_INVARIANTS }; 264 static struct bpf_insn insns[] = { 265 BPF_MOV64_IMM(BPF_REG_0, 0), 266 BPF_EXIT_INSN(), 267 }; 268 int insn_cnt = ARRAY_SIZE(insns), i, fd, flags = 0; 269 LIBBPF_OPTS(bpf_prog_load_opts, opts); 270 271 if (cached_flags >= 0) 272 return cached_flags; 273 274 for (i = 0; i < ARRAY_SIZE(prog_flags); i++) { 275 opts.prog_flags = prog_flags[i]; 276 fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "flag-test", "GPL", 277 insns, insn_cnt, &opts); 278 if (fd >= 0) { 279 flags |= prog_flags[i]; 280 close(fd); 281 } 282 } 283 284 cached_flags = flags; 285 return cached_flags; 286 } 287 288 int bpf_prog_test_load(const char *file, enum bpf_prog_type type, 289 struct bpf_object **pobj, int *prog_fd) 290 { 291 LIBBPF_OPTS(bpf_object_open_opts, opts, 292 .kernel_log_level = extra_prog_load_log_flags, 293 ); 294 struct bpf_object *obj; 295 struct bpf_program *prog; 296 __u32 flags; 297 int err; 298 299 obj = bpf_object__open_file(file, &opts); 300 if (!obj) 301 return -errno; 302 303 prog = bpf_object__next_program(obj, NULL); 304 if (!prog) { 305 err = -ENOENT; 306 goto err_out; 307 } 308 309 if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type) 310 bpf_program__set_type(prog, type); 311 312 flags = bpf_program__flags(prog) | testing_prog_flags(); 313 bpf_program__set_flags(prog, flags); 314 315 err = bpf_object__load(obj); 316 if (err) 317 goto err_out; 318 319 *pobj = obj; 320 *prog_fd = bpf_program__fd(prog); 321 322 return 0; 323 err_out: 324 bpf_object__close(obj); 325 return err; 326 } 327 328 int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, 329 size_t insns_cnt, const char *license, 330 __u32 kern_version, char *log_buf, 331 size_t log_buf_sz) 332 { 333 LIBBPF_OPTS(bpf_prog_load_opts, opts, 334 .kern_version = kern_version, 335 .prog_flags = testing_prog_flags(), 336 .log_level = extra_prog_load_log_flags, 337 .log_buf = log_buf, 338 .log_size = log_buf_sz, 339 ); 340 341 return bpf_prog_load(type, NULL, license, insns, insns_cnt, &opts); 342 } 343 344 __u64 read_perf_max_sample_freq(void) 345 { 346 __u64 sample_freq = 5000; /* fallback to 5000 on error */ 347 FILE *f; 348 349 f = fopen("/proc/sys/kernel/perf_event_max_sample_rate", "r"); 350 if (f == NULL) { 351 printf("Failed to open /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 352 "return default value: 5000\n", -errno); 353 return sample_freq; 354 } 355 if (fscanf(f, "%llu", &sample_freq) != 1) { 356 printf("Failed to parse /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 357 "return default value: 5000\n", -errno); 358 } 359 360 fclose(f); 361 return sample_freq; 362 } 363 364 int finit_module(int fd, const char *param_values, int flags) 365 { 366 return syscall(__NR_finit_module, fd, param_values, flags); 367 } 368 369 int delete_module(const char *name, int flags) 370 { 371 return syscall(__NR_delete_module, name, flags); 372 } 373 374 int try_unload_module(const char *name, int retries, bool verbose) 375 { 376 int ret, cnt = 0; 377 378 if (kern_sync_rcu()) 379 fprintf(stdout, "Failed to trigger kernel-side RCU sync!\n"); 380 381 for (;;) { 382 ret = delete_module(name, 0); 383 if (!ret || errno != EAGAIN) 384 break; 385 if (++cnt > retries) { 386 fprintf(stdout, "Unload of %s timed out\n", name); 387 break; 388 } 389 usleep(100); 390 } 391 392 if (ret) { 393 if (errno == ENOENT) { 394 if (verbose) 395 fprintf(stdout, "%s.ko is already unloaded.\n", name); 396 return -1; 397 } 398 fprintf(stdout, "Failed to unload %s.ko from kernel: %d\n", name, -errno); 399 return -1; 400 } 401 if (verbose) 402 fprintf(stdout, "Successfully unloaded %s.ko.\n", name); 403 return 0; 404 } 405 406 int unload_module(const char *name, bool verbose) 407 { 408 return try_unload_module(name, 10000, verbose); 409 } 410 411 static int __load_module(const char *path, const char *param_values, bool verbose) 412 { 413 int fd; 414 415 if (verbose) 416 fprintf(stdout, "Loading %s...\n", path); 417 418 fd = open(path, O_RDONLY); 419 if (fd < 0) { 420 fprintf(stdout, "Can't find %s kernel module: %d\n", path, -errno); 421 return -ENOENT; 422 } 423 if (finit_module(fd, param_values, 0)) { 424 fprintf(stdout, "Failed to load %s into the kernel: %d\n", path, -errno); 425 close(fd); 426 return -EINVAL; 427 } 428 close(fd); 429 430 if (verbose) 431 fprintf(stdout, "Successfully loaded %s.\n", path); 432 return 0; 433 } 434 435 int load_module_params(const char *path, const char *param_values, bool verbose) 436 { 437 return __load_module(path, param_values, verbose); 438 } 439 440 int load_module(const char *path, bool verbose) 441 { 442 return __load_module(path, "", verbose); 443 } 444 445 int unload_bpf_testmod(bool verbose) 446 { 447 return unload_module("bpf_testmod", verbose); 448 } 449 450 int load_bpf_testmod(bool verbose) 451 { 452 return load_module("bpf_testmod.ko", verbose); 453 } 454 455 /* 456 * Trigger synchronize_rcu() in kernel. 457 */ 458 int kern_sync_rcu(void) 459 { 460 return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, 0, 0); 461 } 462 463 int uffd_block_page(void *fault_addr) 464 { 465 struct uffdio_register uffd_register = {}; 466 struct uffdio_api uffd_api = {}; 467 int uffd; 468 469 uffd = syscall(__NR_userfaultfd, O_CLOEXEC); 470 if (uffd < 0) 471 return -errno; 472 473 uffd_api.api = UFFD_API; 474 uffd_api.features = 0; 475 if (ioctl(uffd, UFFDIO_API, &uffd_api)) { 476 close(uffd); 477 return -1; 478 } 479 480 uffd_register.range.start = (unsigned long)fault_addr; 481 uffd_register.range.len = getpagesize(); 482 uffd_register.mode = UFFDIO_REGISTER_MODE_MISSING; 483 if (ioctl(uffd, UFFDIO_REGISTER, &uffd_register)) { 484 close(uffd); 485 return -1; 486 } 487 return uffd; 488 } 489 490 int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt) 491 { 492 __u32 buf_element_size = sizeof(struct bpf_insn); 493 struct bpf_prog_info info = {}; 494 __u32 info_len = sizeof(info); 495 __u32 xlated_prog_len; 496 497 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 498 perror("bpf_prog_get_info_by_fd failed"); 499 return -1; 500 } 501 502 xlated_prog_len = info.xlated_prog_len; 503 if (xlated_prog_len % buf_element_size) { 504 printf("Program length %u is not multiple of %u\n", 505 xlated_prog_len, buf_element_size); 506 return -1; 507 } 508 509 *cnt = xlated_prog_len / buf_element_size; 510 *buf = calloc(*cnt, buf_element_size); 511 if (!*buf) { 512 perror("can't allocate xlated program buffer"); 513 return -ENOMEM; 514 } 515 516 bzero(&info, sizeof(info)); 517 info.xlated_prog_len = xlated_prog_len; 518 info.xlated_prog_insns = (__u64)(unsigned long)*buf; 519 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 520 perror("second bpf_prog_get_info_by_fd failed"); 521 goto out_free_buf; 522 } 523 524 return 0; 525 526 out_free_buf: 527 free(*buf); 528 *buf = NULL; 529 return -1; 530 } 531 532 bool is_jit_enabled(void) 533 { 534 const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable"; 535 bool enabled = false; 536 int sysctl_fd; 537 538 sysctl_fd = open(jit_sysctl, O_RDONLY); 539 if (sysctl_fd != -1) { 540 char tmpc; 541 542 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1) 543 enabled = (tmpc != '0'); 544 close(sysctl_fd); 545 } 546 547 return enabled; 548 } 549 550 int stack_mprotect(void) 551 { 552 void *buf; 553 long sz; 554 int ret; 555 556 sz = sysconf(_SC_PAGESIZE); 557 if (sz < 0) 558 return sz; 559 560 buf = alloca(sz * 3); 561 ret = mprotect((void *)(((unsigned long)(buf + sz)) & ~(sz - 1)), sz, 562 PROT_READ | PROT_WRITE | PROT_EXEC); 563 return ret; 564 } 565