1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (C) 2019 Netronome Systems, Inc. */ 3 /* Copyright (C) 2020 Facebook, Inc. */ 4 #include <ctype.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <errno.h> 8 #include <bpf/bpf.h> 9 #include <bpf/libbpf.h> 10 #include "disasm.h" 11 #include "test_progs.h" 12 #include "testing_helpers.h" 13 #include <linux/membarrier.h> 14 15 int parse_num_list(const char *s, bool **num_set, int *num_set_len) 16 { 17 int i, set_len = 0, new_len, num, start = 0, end = -1; 18 bool *set = NULL, *tmp, parsing_end = false; 19 char *next; 20 21 while (s[0]) { 22 errno = 0; 23 num = strtol(s, &next, 10); 24 if (errno) 25 return -errno; 26 27 if (parsing_end) 28 end = num; 29 else 30 start = num; 31 32 if (!parsing_end && *next == '-') { 33 s = next + 1; 34 parsing_end = true; 35 continue; 36 } else if (*next == ',') { 37 parsing_end = false; 38 s = next + 1; 39 end = num; 40 } else if (*next == '\0') { 41 parsing_end = false; 42 s = next; 43 end = num; 44 } else { 45 return -EINVAL; 46 } 47 48 if (start > end) 49 return -EINVAL; 50 51 if (end + 1 > set_len) { 52 new_len = end + 1; 53 tmp = realloc(set, new_len); 54 if (!tmp) { 55 free(set); 56 return -ENOMEM; 57 } 58 for (i = set_len; i < start; i++) 59 tmp[i] = false; 60 set = tmp; 61 set_len = new_len; 62 } 63 for (i = start; i <= end; i++) 64 set[i] = true; 65 } 66 67 if (!set || parsing_end) 68 return -EINVAL; 69 70 *num_set = set; 71 *num_set_len = set_len; 72 73 return 0; 74 } 75 76 static int do_insert_test(struct test_filter_set *set, 77 char *test_str, 78 char *subtest_str) 79 { 80 struct test_filter *tmp, *test; 81 char **ctmp; 82 int i; 83 84 for (i = 0; i < set->cnt; i++) { 85 test = &set->tests[i]; 86 87 if (strcmp(test_str, test->name) == 0) { 88 free(test_str); 89 goto subtest; 90 } 91 } 92 93 tmp = realloc(set->tests, sizeof(*test) * (set->cnt + 1)); 94 if (!tmp) 95 return -ENOMEM; 96 97 set->tests = tmp; 98 test = &set->tests[set->cnt]; 99 100 test->name = test_str; 101 test->subtests = NULL; 102 test->subtest_cnt = 0; 103 104 set->cnt++; 105 106 subtest: 107 if (!subtest_str) 108 return 0; 109 110 for (i = 0; i < test->subtest_cnt; i++) { 111 if (strcmp(subtest_str, test->subtests[i]) == 0) { 112 free(subtest_str); 113 return 0; 114 } 115 } 116 117 ctmp = realloc(test->subtests, 118 sizeof(*test->subtests) * (test->subtest_cnt + 1)); 119 if (!ctmp) 120 return -ENOMEM; 121 122 test->subtests = ctmp; 123 test->subtests[test->subtest_cnt] = subtest_str; 124 125 test->subtest_cnt++; 126 127 return 0; 128 } 129 130 static int insert_test(struct test_filter_set *set, 131 char *test_spec, 132 bool is_glob_pattern) 133 { 134 char *pattern, *subtest_str, *ext_test_str, *ext_subtest_str = NULL; 135 int glob_chars = 0; 136 137 if (is_glob_pattern) { 138 pattern = "%s"; 139 } else { 140 pattern = "*%s*"; 141 glob_chars = 2; 142 } 143 144 subtest_str = strchr(test_spec, '/'); 145 if (subtest_str) { 146 *subtest_str = '\0'; 147 subtest_str += 1; 148 } 149 150 ext_test_str = malloc(strlen(test_spec) + glob_chars + 1); 151 if (!ext_test_str) 152 goto err; 153 154 sprintf(ext_test_str, pattern, test_spec); 155 156 if (subtest_str) { 157 ext_subtest_str = malloc(strlen(subtest_str) + glob_chars + 1); 158 if (!ext_subtest_str) 159 goto err; 160 161 sprintf(ext_subtest_str, pattern, subtest_str); 162 } 163 164 return do_insert_test(set, ext_test_str, ext_subtest_str); 165 166 err: 167 free(ext_test_str); 168 free(ext_subtest_str); 169 170 return -ENOMEM; 171 } 172 173 int parse_test_list_file(const char *path, 174 struct test_filter_set *set, 175 bool is_glob_pattern) 176 { 177 char *buf = NULL, *capture_start, *capture_end, *scan_end; 178 size_t buflen = 0; 179 int err = 0; 180 FILE *f; 181 182 f = fopen(path, "r"); 183 if (!f) { 184 err = -errno; 185 fprintf(stderr, "Failed to open '%s': %d\n", path, err); 186 return err; 187 } 188 189 while (getline(&buf, &buflen, f) != -1) { 190 capture_start = buf; 191 192 while (isspace(*capture_start)) 193 ++capture_start; 194 195 capture_end = capture_start; 196 scan_end = capture_start; 197 198 while (*scan_end && *scan_end != '#') { 199 if (!isspace(*scan_end)) 200 capture_end = scan_end; 201 202 ++scan_end; 203 } 204 205 if (capture_end == capture_start) 206 continue; 207 208 *(++capture_end) = '\0'; 209 210 err = insert_test(set, capture_start, is_glob_pattern); 211 if (err) 212 break; 213 } 214 215 fclose(f); 216 return err; 217 } 218 219 int parse_test_list(const char *s, 220 struct test_filter_set *set, 221 bool is_glob_pattern) 222 { 223 char *input, *state = NULL, *test_spec; 224 int err = 0, cnt = 0; 225 226 input = strdup(s); 227 if (!input) 228 return -ENOMEM; 229 230 while ((test_spec = strtok_r(cnt++ ? NULL : input, ",", &state))) { 231 err = insert_test(set, test_spec, is_glob_pattern); 232 if (err) 233 break; 234 } 235 236 free(input); 237 return err; 238 } 239 240 __u32 link_info_prog_id(const struct bpf_link *link, struct bpf_link_info *info) 241 { 242 __u32 info_len = sizeof(*info); 243 int err; 244 245 memset(info, 0, sizeof(*info)); 246 err = bpf_link_get_info_by_fd(bpf_link__fd(link), info, &info_len); 247 if (err) { 248 printf("failed to get link info: %d\n", -errno); 249 return 0; 250 } 251 return info->prog_id; 252 } 253 254 int extra_prog_load_log_flags = 0; 255 256 int testing_prog_flags(void) 257 { 258 static int cached_flags = -1; 259 static int prog_flags[] = { BPF_F_TEST_RND_HI32, BPF_F_TEST_REG_INVARIANTS }; 260 static struct bpf_insn insns[] = { 261 BPF_MOV64_IMM(BPF_REG_0, 0), 262 BPF_EXIT_INSN(), 263 }; 264 int insn_cnt = ARRAY_SIZE(insns), i, fd, flags = 0; 265 LIBBPF_OPTS(bpf_prog_load_opts, opts); 266 267 if (cached_flags >= 0) 268 return cached_flags; 269 270 for (i = 0; i < ARRAY_SIZE(prog_flags); i++) { 271 opts.prog_flags = prog_flags[i]; 272 fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "flag-test", "GPL", 273 insns, insn_cnt, &opts); 274 if (fd >= 0) { 275 flags |= prog_flags[i]; 276 close(fd); 277 } 278 } 279 280 cached_flags = flags; 281 return cached_flags; 282 } 283 284 int bpf_prog_test_load(const char *file, enum bpf_prog_type type, 285 struct bpf_object **pobj, int *prog_fd) 286 { 287 LIBBPF_OPTS(bpf_object_open_opts, opts, 288 .kernel_log_level = extra_prog_load_log_flags, 289 ); 290 struct bpf_object *obj; 291 struct bpf_program *prog; 292 __u32 flags; 293 int err; 294 295 obj = bpf_object__open_file(file, &opts); 296 if (!obj) 297 return -errno; 298 299 prog = bpf_object__next_program(obj, NULL); 300 if (!prog) { 301 err = -ENOENT; 302 goto err_out; 303 } 304 305 if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type) 306 bpf_program__set_type(prog, type); 307 308 flags = bpf_program__flags(prog) | testing_prog_flags(); 309 bpf_program__set_flags(prog, flags); 310 311 err = bpf_object__load(obj); 312 if (err) 313 goto err_out; 314 315 *pobj = obj; 316 *prog_fd = bpf_program__fd(prog); 317 318 return 0; 319 err_out: 320 bpf_object__close(obj); 321 return err; 322 } 323 324 int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, 325 size_t insns_cnt, const char *license, 326 __u32 kern_version, char *log_buf, 327 size_t log_buf_sz) 328 { 329 LIBBPF_OPTS(bpf_prog_load_opts, opts, 330 .kern_version = kern_version, 331 .prog_flags = testing_prog_flags(), 332 .log_level = extra_prog_load_log_flags, 333 .log_buf = log_buf, 334 .log_size = log_buf_sz, 335 ); 336 337 return bpf_prog_load(type, NULL, license, insns, insns_cnt, &opts); 338 } 339 340 __u64 read_perf_max_sample_freq(void) 341 { 342 __u64 sample_freq = 5000; /* fallback to 5000 on error */ 343 FILE *f; 344 345 f = fopen("/proc/sys/kernel/perf_event_max_sample_rate", "r"); 346 if (f == NULL) { 347 printf("Failed to open /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 348 "return default value: 5000\n", -errno); 349 return sample_freq; 350 } 351 if (fscanf(f, "%llu", &sample_freq) != 1) { 352 printf("Failed to parse /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 353 "return default value: 5000\n", -errno); 354 } 355 356 fclose(f); 357 return sample_freq; 358 } 359 360 int finit_module(int fd, const char *param_values, int flags) 361 { 362 return syscall(__NR_finit_module, fd, param_values, flags); 363 } 364 365 int delete_module(const char *name, int flags) 366 { 367 return syscall(__NR_delete_module, name, flags); 368 } 369 370 int unload_module(const char *name, bool verbose) 371 { 372 int ret, cnt = 0; 373 374 if (kern_sync_rcu()) 375 fprintf(stdout, "Failed to trigger kernel-side RCU sync!\n"); 376 377 for (;;) { 378 ret = delete_module(name, 0); 379 if (!ret || errno != EAGAIN) 380 break; 381 if (++cnt > 10000) { 382 fprintf(stdout, "Unload of %s timed out\n", name); 383 break; 384 } 385 usleep(100); 386 } 387 388 if (ret) { 389 if (errno == ENOENT) { 390 if (verbose) 391 fprintf(stdout, "%s.ko is already unloaded.\n", name); 392 return -1; 393 } 394 fprintf(stdout, "Failed to unload %s.ko from kernel: %d\n", name, -errno); 395 return -1; 396 } 397 if (verbose) 398 fprintf(stdout, "Successfully unloaded %s.ko.\n", name); 399 return 0; 400 } 401 402 int load_module(const char *path, bool verbose) 403 { 404 int fd; 405 406 if (verbose) 407 fprintf(stdout, "Loading %s...\n", path); 408 409 fd = open(path, O_RDONLY); 410 if (fd < 0) { 411 fprintf(stdout, "Can't find %s kernel module: %d\n", path, -errno); 412 return -ENOENT; 413 } 414 if (finit_module(fd, "", 0)) { 415 fprintf(stdout, "Failed to load %s into the kernel: %d\n", path, -errno); 416 close(fd); 417 return -EINVAL; 418 } 419 close(fd); 420 421 if (verbose) 422 fprintf(stdout, "Successfully loaded %s.\n", path); 423 return 0; 424 } 425 426 int unload_bpf_testmod(bool verbose) 427 { 428 return unload_module("bpf_testmod", verbose); 429 } 430 431 int load_bpf_testmod(bool verbose) 432 { 433 return load_module("bpf_testmod.ko", verbose); 434 } 435 436 /* 437 * Trigger synchronize_rcu() in kernel. 438 */ 439 int kern_sync_rcu(void) 440 { 441 return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, 0, 0); 442 } 443 444 int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt) 445 { 446 __u32 buf_element_size = sizeof(struct bpf_insn); 447 struct bpf_prog_info info = {}; 448 __u32 info_len = sizeof(info); 449 __u32 xlated_prog_len; 450 451 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 452 perror("bpf_prog_get_info_by_fd failed"); 453 return -1; 454 } 455 456 xlated_prog_len = info.xlated_prog_len; 457 if (xlated_prog_len % buf_element_size) { 458 printf("Program length %u is not multiple of %u\n", 459 xlated_prog_len, buf_element_size); 460 return -1; 461 } 462 463 *cnt = xlated_prog_len / buf_element_size; 464 *buf = calloc(*cnt, buf_element_size); 465 if (!*buf) { 466 perror("can't allocate xlated program buffer"); 467 return -ENOMEM; 468 } 469 470 bzero(&info, sizeof(info)); 471 info.xlated_prog_len = xlated_prog_len; 472 info.xlated_prog_insns = (__u64)(unsigned long)*buf; 473 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 474 perror("second bpf_prog_get_info_by_fd failed"); 475 goto out_free_buf; 476 } 477 478 return 0; 479 480 out_free_buf: 481 free(*buf); 482 *buf = NULL; 483 return -1; 484 } 485 486 bool is_jit_enabled(void) 487 { 488 const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable"; 489 bool enabled = false; 490 int sysctl_fd; 491 492 sysctl_fd = open(jit_sysctl, O_RDONLY); 493 if (sysctl_fd != -1) { 494 char tmpc; 495 496 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1) 497 enabled = (tmpc != '0'); 498 close(sysctl_fd); 499 } 500 501 return enabled; 502 } 503