1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (C) 2019 Netronome Systems, Inc. */ 3 /* Copyright (C) 2020 Facebook, Inc. */ 4 #include <ctype.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <errno.h> 8 #include <sys/mman.h> 9 #include <alloca.h> 10 #include <bpf/bpf.h> 11 #include <bpf/libbpf.h> 12 #include "disasm.h" 13 #include "test_progs.h" 14 #include "testing_helpers.h" 15 #include <linux/membarrier.h> 16 17 int parse_num_list(const char *s, bool **num_set, int *num_set_len) 18 { 19 int i, set_len = 0, new_len, num, start = 0, end = -1; 20 bool *set = NULL, *tmp, parsing_end = false; 21 char *next; 22 23 while (s[0]) { 24 errno = 0; 25 num = strtol(s, &next, 10); 26 if (errno) 27 return -errno; 28 29 if (parsing_end) 30 end = num; 31 else 32 start = num; 33 34 if (!parsing_end && *next == '-') { 35 s = next + 1; 36 parsing_end = true; 37 continue; 38 } else if (*next == ',') { 39 parsing_end = false; 40 s = next + 1; 41 end = num; 42 } else if (*next == '\0') { 43 parsing_end = false; 44 s = next; 45 end = num; 46 } else { 47 return -EINVAL; 48 } 49 50 if (start > end) 51 return -EINVAL; 52 53 if (end + 1 > set_len) { 54 new_len = end + 1; 55 tmp = realloc(set, new_len); 56 if (!tmp) { 57 free(set); 58 return -ENOMEM; 59 } 60 for (i = set_len; i < start; i++) 61 tmp[i] = false; 62 set = tmp; 63 set_len = new_len; 64 } 65 for (i = start; i <= end; i++) 66 set[i] = true; 67 } 68 69 if (!set || parsing_end) 70 return -EINVAL; 71 72 *num_set = set; 73 *num_set_len = set_len; 74 75 return 0; 76 } 77 78 static int do_insert_test(struct test_filter_set *set, 79 char *test_str, 80 char *subtest_str) 81 { 82 struct test_filter *tmp, *test; 83 char **ctmp; 84 int i; 85 86 for (i = 0; i < set->cnt; i++) { 87 test = &set->tests[i]; 88 89 if (strcmp(test_str, test->name) == 0) { 90 free(test_str); 91 goto subtest; 92 } 93 } 94 95 tmp = realloc(set->tests, sizeof(*test) * (set->cnt + 1)); 96 if (!tmp) 97 return -ENOMEM; 98 99 set->tests = tmp; 100 test = &set->tests[set->cnt]; 101 102 test->name = test_str; 103 test->subtests = NULL; 104 test->subtest_cnt = 0; 105 106 set->cnt++; 107 108 subtest: 109 if (!subtest_str) 110 return 0; 111 112 for (i = 0; i < test->subtest_cnt; i++) { 113 if (strcmp(subtest_str, test->subtests[i]) == 0) { 114 free(subtest_str); 115 return 0; 116 } 117 } 118 119 ctmp = realloc(test->subtests, 120 sizeof(*test->subtests) * (test->subtest_cnt + 1)); 121 if (!ctmp) 122 return -ENOMEM; 123 124 test->subtests = ctmp; 125 test->subtests[test->subtest_cnt] = subtest_str; 126 127 test->subtest_cnt++; 128 129 return 0; 130 } 131 132 static int insert_test(struct test_filter_set *set, 133 char *test_spec, 134 bool is_glob_pattern) 135 { 136 char *pattern, *subtest_str, *ext_test_str, *ext_subtest_str = NULL; 137 int glob_chars = 0; 138 139 if (is_glob_pattern) { 140 pattern = "%s"; 141 } else { 142 pattern = "*%s*"; 143 glob_chars = 2; 144 } 145 146 subtest_str = strchr(test_spec, '/'); 147 if (subtest_str) { 148 *subtest_str = '\0'; 149 subtest_str += 1; 150 } 151 152 ext_test_str = malloc(strlen(test_spec) + glob_chars + 1); 153 if (!ext_test_str) 154 goto err; 155 156 sprintf(ext_test_str, pattern, test_spec); 157 158 if (subtest_str) { 159 ext_subtest_str = malloc(strlen(subtest_str) + glob_chars + 1); 160 if (!ext_subtest_str) 161 goto err; 162 163 sprintf(ext_subtest_str, pattern, subtest_str); 164 } 165 166 return do_insert_test(set, ext_test_str, ext_subtest_str); 167 168 err: 169 free(ext_test_str); 170 free(ext_subtest_str); 171 172 return -ENOMEM; 173 } 174 175 int parse_test_list_file(const char *path, 176 struct test_filter_set *set, 177 bool is_glob_pattern) 178 { 179 char *buf = NULL, *capture_start, *capture_end, *scan_end; 180 size_t buflen = 0; 181 int err = 0; 182 FILE *f; 183 184 f = fopen(path, "r"); 185 if (!f) { 186 err = -errno; 187 fprintf(stderr, "Failed to open '%s': %d\n", path, err); 188 return err; 189 } 190 191 while (getline(&buf, &buflen, f) != -1) { 192 capture_start = buf; 193 194 while (isspace(*capture_start)) 195 ++capture_start; 196 197 capture_end = capture_start; 198 scan_end = capture_start; 199 200 while (*scan_end && *scan_end != '#') { 201 if (!isspace(*scan_end)) 202 capture_end = scan_end; 203 204 ++scan_end; 205 } 206 207 if (capture_end == capture_start) 208 continue; 209 210 *(++capture_end) = '\0'; 211 212 err = insert_test(set, capture_start, is_glob_pattern); 213 if (err) 214 break; 215 } 216 217 free(buf); 218 fclose(f); 219 return err; 220 } 221 222 int parse_test_list(const char *s, 223 struct test_filter_set *set, 224 bool is_glob_pattern) 225 { 226 char *input, *state = NULL, *test_spec; 227 int err = 0, cnt = 0; 228 229 input = strdup(s); 230 if (!input) 231 return -ENOMEM; 232 233 while ((test_spec = strtok_r(cnt++ ? NULL : input, ",", &state))) { 234 err = insert_test(set, test_spec, is_glob_pattern); 235 if (err) 236 break; 237 } 238 239 free(input); 240 return err; 241 } 242 243 __u32 link_info_prog_id(const struct bpf_link *link, struct bpf_link_info *info) 244 { 245 __u32 info_len = sizeof(*info); 246 int err; 247 248 memset(info, 0, sizeof(*info)); 249 err = bpf_link_get_info_by_fd(bpf_link__fd(link), info, &info_len); 250 if (err) { 251 printf("failed to get link info: %d\n", -errno); 252 return 0; 253 } 254 return info->prog_id; 255 } 256 257 int extra_prog_load_log_flags = 0; 258 259 int testing_prog_flags(void) 260 { 261 static int cached_flags = -1; 262 static int prog_flags[] = { BPF_F_TEST_RND_HI32, BPF_F_TEST_REG_INVARIANTS }; 263 static struct bpf_insn insns[] = { 264 BPF_MOV64_IMM(BPF_REG_0, 0), 265 BPF_EXIT_INSN(), 266 }; 267 int insn_cnt = ARRAY_SIZE(insns), i, fd, flags = 0; 268 LIBBPF_OPTS(bpf_prog_load_opts, opts); 269 270 if (cached_flags >= 0) 271 return cached_flags; 272 273 for (i = 0; i < ARRAY_SIZE(prog_flags); i++) { 274 opts.prog_flags = prog_flags[i]; 275 fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "flag-test", "GPL", 276 insns, insn_cnt, &opts); 277 if (fd >= 0) { 278 flags |= prog_flags[i]; 279 close(fd); 280 } 281 } 282 283 cached_flags = flags; 284 return cached_flags; 285 } 286 287 int bpf_prog_test_load(const char *file, enum bpf_prog_type type, 288 struct bpf_object **pobj, int *prog_fd) 289 { 290 LIBBPF_OPTS(bpf_object_open_opts, opts, 291 .kernel_log_level = extra_prog_load_log_flags, 292 ); 293 struct bpf_object *obj; 294 struct bpf_program *prog; 295 __u32 flags; 296 int err; 297 298 obj = bpf_object__open_file(file, &opts); 299 if (!obj) 300 return -errno; 301 302 prog = bpf_object__next_program(obj, NULL); 303 if (!prog) { 304 err = -ENOENT; 305 goto err_out; 306 } 307 308 if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type) 309 bpf_program__set_type(prog, type); 310 311 flags = bpf_program__flags(prog) | testing_prog_flags(); 312 bpf_program__set_flags(prog, flags); 313 314 err = bpf_object__load(obj); 315 if (err) 316 goto err_out; 317 318 *pobj = obj; 319 *prog_fd = bpf_program__fd(prog); 320 321 return 0; 322 err_out: 323 bpf_object__close(obj); 324 return err; 325 } 326 327 int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, 328 size_t insns_cnt, const char *license, 329 __u32 kern_version, char *log_buf, 330 size_t log_buf_sz) 331 { 332 LIBBPF_OPTS(bpf_prog_load_opts, opts, 333 .kern_version = kern_version, 334 .prog_flags = testing_prog_flags(), 335 .log_level = extra_prog_load_log_flags, 336 .log_buf = log_buf, 337 .log_size = log_buf_sz, 338 ); 339 340 return bpf_prog_load(type, NULL, license, insns, insns_cnt, &opts); 341 } 342 343 __u64 read_perf_max_sample_freq(void) 344 { 345 __u64 sample_freq = 5000; /* fallback to 5000 on error */ 346 FILE *f; 347 348 f = fopen("/proc/sys/kernel/perf_event_max_sample_rate", "r"); 349 if (f == NULL) { 350 printf("Failed to open /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 351 "return default value: 5000\n", -errno); 352 return sample_freq; 353 } 354 if (fscanf(f, "%llu", &sample_freq) != 1) { 355 printf("Failed to parse /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 356 "return default value: 5000\n", -errno); 357 } 358 359 fclose(f); 360 return sample_freq; 361 } 362 363 int finit_module(int fd, const char *param_values, int flags) 364 { 365 return syscall(__NR_finit_module, fd, param_values, flags); 366 } 367 368 int delete_module(const char *name, int flags) 369 { 370 return syscall(__NR_delete_module, name, flags); 371 } 372 373 int try_unload_module(const char *name, int retries, bool verbose) 374 { 375 int ret, cnt = 0; 376 377 if (kern_sync_rcu()) 378 fprintf(stdout, "Failed to trigger kernel-side RCU sync!\n"); 379 380 for (;;) { 381 ret = delete_module(name, 0); 382 if (!ret || errno != EAGAIN) 383 break; 384 if (++cnt > retries) { 385 fprintf(stdout, "Unload of %s timed out\n", name); 386 break; 387 } 388 usleep(100); 389 } 390 391 if (ret) { 392 if (errno == ENOENT) { 393 if (verbose) 394 fprintf(stdout, "%s.ko is already unloaded.\n", name); 395 return -1; 396 } 397 fprintf(stdout, "Failed to unload %s.ko from kernel: %d\n", name, -errno); 398 return -1; 399 } 400 if (verbose) 401 fprintf(stdout, "Successfully unloaded %s.ko.\n", name); 402 return 0; 403 } 404 405 int unload_module(const char *name, bool verbose) 406 { 407 return try_unload_module(name, 10000, verbose); 408 } 409 410 static int __load_module(const char *path, const char *param_values, bool verbose) 411 { 412 int fd; 413 414 if (verbose) 415 fprintf(stdout, "Loading %s...\n", path); 416 417 fd = open(path, O_RDONLY); 418 if (fd < 0) { 419 fprintf(stdout, "Can't find %s kernel module: %d\n", path, -errno); 420 return -ENOENT; 421 } 422 if (finit_module(fd, param_values, 0)) { 423 fprintf(stdout, "Failed to load %s into the kernel: %d\n", path, -errno); 424 close(fd); 425 return -EINVAL; 426 } 427 close(fd); 428 429 if (verbose) 430 fprintf(stdout, "Successfully loaded %s.\n", path); 431 return 0; 432 } 433 434 int load_module_params(const char *path, const char *param_values, bool verbose) 435 { 436 return __load_module(path, param_values, verbose); 437 } 438 439 int load_module(const char *path, bool verbose) 440 { 441 return __load_module(path, "", verbose); 442 } 443 444 int unload_bpf_testmod(bool verbose) 445 { 446 return unload_module("bpf_testmod", verbose); 447 } 448 449 int load_bpf_testmod(bool verbose) 450 { 451 return load_module("bpf_testmod.ko", verbose); 452 } 453 454 /* 455 * Trigger synchronize_rcu() in kernel. 456 */ 457 int kern_sync_rcu(void) 458 { 459 return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, 0, 0); 460 } 461 462 int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt) 463 { 464 __u32 buf_element_size = sizeof(struct bpf_insn); 465 struct bpf_prog_info info = {}; 466 __u32 info_len = sizeof(info); 467 __u32 xlated_prog_len; 468 469 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 470 perror("bpf_prog_get_info_by_fd failed"); 471 return -1; 472 } 473 474 xlated_prog_len = info.xlated_prog_len; 475 if (xlated_prog_len % buf_element_size) { 476 printf("Program length %u is not multiple of %u\n", 477 xlated_prog_len, buf_element_size); 478 return -1; 479 } 480 481 *cnt = xlated_prog_len / buf_element_size; 482 *buf = calloc(*cnt, buf_element_size); 483 if (!*buf) { 484 perror("can't allocate xlated program buffer"); 485 return -ENOMEM; 486 } 487 488 bzero(&info, sizeof(info)); 489 info.xlated_prog_len = xlated_prog_len; 490 info.xlated_prog_insns = (__u64)(unsigned long)*buf; 491 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 492 perror("second bpf_prog_get_info_by_fd failed"); 493 goto out_free_buf; 494 } 495 496 return 0; 497 498 out_free_buf: 499 free(*buf); 500 *buf = NULL; 501 return -1; 502 } 503 504 bool is_jit_enabled(void) 505 { 506 const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable"; 507 bool enabled = false; 508 int sysctl_fd; 509 510 sysctl_fd = open(jit_sysctl, O_RDONLY); 511 if (sysctl_fd != -1) { 512 char tmpc; 513 514 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1) 515 enabled = (tmpc != '0'); 516 close(sysctl_fd); 517 } 518 519 return enabled; 520 } 521 522 int stack_mprotect(void) 523 { 524 void *buf; 525 long sz; 526 int ret; 527 528 sz = sysconf(_SC_PAGESIZE); 529 if (sz < 0) 530 return sz; 531 532 buf = alloca(sz * 3); 533 ret = mprotect((void *)(((unsigned long)(buf + sz)) & ~(sz - 1)), sz, 534 PROT_READ | PROT_WRITE | PROT_EXEC); 535 return ret; 536 } 537