1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (C) 2019 Netronome Systems, Inc. */ 3 /* Copyright (C) 2020 Facebook, Inc. */ 4 #include <ctype.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <errno.h> 8 #include <bpf/bpf.h> 9 #include <bpf/libbpf.h> 10 #include "disasm.h" 11 #include "test_progs.h" 12 #include "testing_helpers.h" 13 #include <linux/membarrier.h> 14 15 int parse_num_list(const char *s, bool **num_set, int *num_set_len) 16 { 17 int i, set_len = 0, new_len, num, start = 0, end = -1; 18 bool *set = NULL, *tmp, parsing_end = false; 19 char *next; 20 21 while (s[0]) { 22 errno = 0; 23 num = strtol(s, &next, 10); 24 if (errno) 25 return -errno; 26 27 if (parsing_end) 28 end = num; 29 else 30 start = num; 31 32 if (!parsing_end && *next == '-') { 33 s = next + 1; 34 parsing_end = true; 35 continue; 36 } else if (*next == ',') { 37 parsing_end = false; 38 s = next + 1; 39 end = num; 40 } else if (*next == '\0') { 41 parsing_end = false; 42 s = next; 43 end = num; 44 } else { 45 return -EINVAL; 46 } 47 48 if (start > end) 49 return -EINVAL; 50 51 if (end + 1 > set_len) { 52 new_len = end + 1; 53 tmp = realloc(set, new_len); 54 if (!tmp) { 55 free(set); 56 return -ENOMEM; 57 } 58 for (i = set_len; i < start; i++) 59 tmp[i] = false; 60 set = tmp; 61 set_len = new_len; 62 } 63 for (i = start; i <= end; i++) 64 set[i] = true; 65 } 66 67 if (!set || parsing_end) 68 return -EINVAL; 69 70 *num_set = set; 71 *num_set_len = set_len; 72 73 return 0; 74 } 75 76 static int do_insert_test(struct test_filter_set *set, 77 char *test_str, 78 char *subtest_str) 79 { 80 struct test_filter *tmp, *test; 81 char **ctmp; 82 int i; 83 84 for (i = 0; i < set->cnt; i++) { 85 test = &set->tests[i]; 86 87 if (strcmp(test_str, test->name) == 0) { 88 free(test_str); 89 goto subtest; 90 } 91 } 92 93 tmp = realloc(set->tests, sizeof(*test) * (set->cnt + 1)); 94 if (!tmp) 95 return -ENOMEM; 96 97 set->tests = tmp; 98 test = &set->tests[set->cnt]; 99 100 test->name = test_str; 101 test->subtests = NULL; 102 test->subtest_cnt = 0; 103 104 set->cnt++; 105 106 subtest: 107 if (!subtest_str) 108 return 0; 109 110 for (i = 0; i < test->subtest_cnt; i++) { 111 if (strcmp(subtest_str, test->subtests[i]) == 0) { 112 free(subtest_str); 113 return 0; 114 } 115 } 116 117 ctmp = realloc(test->subtests, 118 sizeof(*test->subtests) * (test->subtest_cnt + 1)); 119 if (!ctmp) 120 return -ENOMEM; 121 122 test->subtests = ctmp; 123 test->subtests[test->subtest_cnt] = subtest_str; 124 125 test->subtest_cnt++; 126 127 return 0; 128 } 129 130 static int insert_test(struct test_filter_set *set, 131 char *test_spec, 132 bool is_glob_pattern) 133 { 134 char *pattern, *subtest_str, *ext_test_str, *ext_subtest_str = NULL; 135 int glob_chars = 0; 136 137 if (is_glob_pattern) { 138 pattern = "%s"; 139 } else { 140 pattern = "*%s*"; 141 glob_chars = 2; 142 } 143 144 subtest_str = strchr(test_spec, '/'); 145 if (subtest_str) { 146 *subtest_str = '\0'; 147 subtest_str += 1; 148 } 149 150 ext_test_str = malloc(strlen(test_spec) + glob_chars + 1); 151 if (!ext_test_str) 152 goto err; 153 154 sprintf(ext_test_str, pattern, test_spec); 155 156 if (subtest_str) { 157 ext_subtest_str = malloc(strlen(subtest_str) + glob_chars + 1); 158 if (!ext_subtest_str) 159 goto err; 160 161 sprintf(ext_subtest_str, pattern, subtest_str); 162 } 163 164 return do_insert_test(set, ext_test_str, ext_subtest_str); 165 166 err: 167 free(ext_test_str); 168 free(ext_subtest_str); 169 170 return -ENOMEM; 171 } 172 173 int parse_test_list_file(const char *path, 174 struct test_filter_set *set, 175 bool is_glob_pattern) 176 { 177 char *buf = NULL, *capture_start, *capture_end, *scan_end; 178 size_t buflen = 0; 179 int err = 0; 180 FILE *f; 181 182 f = fopen(path, "r"); 183 if (!f) { 184 err = -errno; 185 fprintf(stderr, "Failed to open '%s': %d\n", path, err); 186 return err; 187 } 188 189 while (getline(&buf, &buflen, f) != -1) { 190 capture_start = buf; 191 192 while (isspace(*capture_start)) 193 ++capture_start; 194 195 capture_end = capture_start; 196 scan_end = capture_start; 197 198 while (*scan_end && *scan_end != '#') { 199 if (!isspace(*scan_end)) 200 capture_end = scan_end; 201 202 ++scan_end; 203 } 204 205 if (capture_end == capture_start) 206 continue; 207 208 *(++capture_end) = '\0'; 209 210 err = insert_test(set, capture_start, is_glob_pattern); 211 if (err) 212 break; 213 } 214 215 free(buf); 216 fclose(f); 217 return err; 218 } 219 220 int parse_test_list(const char *s, 221 struct test_filter_set *set, 222 bool is_glob_pattern) 223 { 224 char *input, *state = NULL, *test_spec; 225 int err = 0, cnt = 0; 226 227 input = strdup(s); 228 if (!input) 229 return -ENOMEM; 230 231 while ((test_spec = strtok_r(cnt++ ? NULL : input, ",", &state))) { 232 err = insert_test(set, test_spec, is_glob_pattern); 233 if (err) 234 break; 235 } 236 237 free(input); 238 return err; 239 } 240 241 __u32 link_info_prog_id(const struct bpf_link *link, struct bpf_link_info *info) 242 { 243 __u32 info_len = sizeof(*info); 244 int err; 245 246 memset(info, 0, sizeof(*info)); 247 err = bpf_link_get_info_by_fd(bpf_link__fd(link), info, &info_len); 248 if (err) { 249 printf("failed to get link info: %d\n", -errno); 250 return 0; 251 } 252 return info->prog_id; 253 } 254 255 int extra_prog_load_log_flags = 0; 256 257 int testing_prog_flags(void) 258 { 259 static int cached_flags = -1; 260 static int prog_flags[] = { BPF_F_TEST_RND_HI32, BPF_F_TEST_REG_INVARIANTS }; 261 static struct bpf_insn insns[] = { 262 BPF_MOV64_IMM(BPF_REG_0, 0), 263 BPF_EXIT_INSN(), 264 }; 265 int insn_cnt = ARRAY_SIZE(insns), i, fd, flags = 0; 266 LIBBPF_OPTS(bpf_prog_load_opts, opts); 267 268 if (cached_flags >= 0) 269 return cached_flags; 270 271 for (i = 0; i < ARRAY_SIZE(prog_flags); i++) { 272 opts.prog_flags = prog_flags[i]; 273 fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "flag-test", "GPL", 274 insns, insn_cnt, &opts); 275 if (fd >= 0) { 276 flags |= prog_flags[i]; 277 close(fd); 278 } 279 } 280 281 cached_flags = flags; 282 return cached_flags; 283 } 284 285 int bpf_prog_test_load(const char *file, enum bpf_prog_type type, 286 struct bpf_object **pobj, int *prog_fd) 287 { 288 LIBBPF_OPTS(bpf_object_open_opts, opts, 289 .kernel_log_level = extra_prog_load_log_flags, 290 ); 291 struct bpf_object *obj; 292 struct bpf_program *prog; 293 __u32 flags; 294 int err; 295 296 obj = bpf_object__open_file(file, &opts); 297 if (!obj) 298 return -errno; 299 300 prog = bpf_object__next_program(obj, NULL); 301 if (!prog) { 302 err = -ENOENT; 303 goto err_out; 304 } 305 306 if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type) 307 bpf_program__set_type(prog, type); 308 309 flags = bpf_program__flags(prog) | testing_prog_flags(); 310 bpf_program__set_flags(prog, flags); 311 312 err = bpf_object__load(obj); 313 if (err) 314 goto err_out; 315 316 *pobj = obj; 317 *prog_fd = bpf_program__fd(prog); 318 319 return 0; 320 err_out: 321 bpf_object__close(obj); 322 return err; 323 } 324 325 int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, 326 size_t insns_cnt, const char *license, 327 __u32 kern_version, char *log_buf, 328 size_t log_buf_sz) 329 { 330 LIBBPF_OPTS(bpf_prog_load_opts, opts, 331 .kern_version = kern_version, 332 .prog_flags = testing_prog_flags(), 333 .log_level = extra_prog_load_log_flags, 334 .log_buf = log_buf, 335 .log_size = log_buf_sz, 336 ); 337 338 return bpf_prog_load(type, NULL, license, insns, insns_cnt, &opts); 339 } 340 341 __u64 read_perf_max_sample_freq(void) 342 { 343 __u64 sample_freq = 5000; /* fallback to 5000 on error */ 344 FILE *f; 345 346 f = fopen("/proc/sys/kernel/perf_event_max_sample_rate", "r"); 347 if (f == NULL) { 348 printf("Failed to open /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 349 "return default value: 5000\n", -errno); 350 return sample_freq; 351 } 352 if (fscanf(f, "%llu", &sample_freq) != 1) { 353 printf("Failed to parse /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 354 "return default value: 5000\n", -errno); 355 } 356 357 fclose(f); 358 return sample_freq; 359 } 360 361 int finit_module(int fd, const char *param_values, int flags) 362 { 363 return syscall(__NR_finit_module, fd, param_values, flags); 364 } 365 366 int delete_module(const char *name, int flags) 367 { 368 return syscall(__NR_delete_module, name, flags); 369 } 370 371 int unload_module(const char *name, bool verbose) 372 { 373 int ret, cnt = 0; 374 375 if (kern_sync_rcu()) 376 fprintf(stdout, "Failed to trigger kernel-side RCU sync!\n"); 377 378 for (;;) { 379 ret = delete_module(name, 0); 380 if (!ret || errno != EAGAIN) 381 break; 382 if (++cnt > 10000) { 383 fprintf(stdout, "Unload of %s timed out\n", name); 384 break; 385 } 386 usleep(100); 387 } 388 389 if (ret) { 390 if (errno == ENOENT) { 391 if (verbose) 392 fprintf(stdout, "%s.ko is already unloaded.\n", name); 393 return -1; 394 } 395 fprintf(stdout, "Failed to unload %s.ko from kernel: %d\n", name, -errno); 396 return -1; 397 } 398 if (verbose) 399 fprintf(stdout, "Successfully unloaded %s.ko.\n", name); 400 return 0; 401 } 402 403 static int __load_module(const char *path, const char *param_values, bool verbose) 404 { 405 int fd; 406 407 if (verbose) 408 fprintf(stdout, "Loading %s...\n", path); 409 410 fd = open(path, O_RDONLY); 411 if (fd < 0) { 412 fprintf(stdout, "Can't find %s kernel module: %d\n", path, -errno); 413 return -ENOENT; 414 } 415 if (finit_module(fd, param_values, 0)) { 416 fprintf(stdout, "Failed to load %s into the kernel: %d\n", path, -errno); 417 close(fd); 418 return -EINVAL; 419 } 420 close(fd); 421 422 if (verbose) 423 fprintf(stdout, "Successfully loaded %s.\n", path); 424 return 0; 425 } 426 427 int load_module_params(const char *path, const char *param_values, bool verbose) 428 { 429 return __load_module(path, param_values, verbose); 430 } 431 432 int load_module(const char *path, bool verbose) 433 { 434 return __load_module(path, "", verbose); 435 } 436 437 int unload_bpf_testmod(bool verbose) 438 { 439 return unload_module("bpf_testmod", verbose); 440 } 441 442 int load_bpf_testmod(bool verbose) 443 { 444 return load_module("bpf_testmod.ko", verbose); 445 } 446 447 /* 448 * Trigger synchronize_rcu() in kernel. 449 */ 450 int kern_sync_rcu(void) 451 { 452 return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, 0, 0); 453 } 454 455 int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt) 456 { 457 __u32 buf_element_size = sizeof(struct bpf_insn); 458 struct bpf_prog_info info = {}; 459 __u32 info_len = sizeof(info); 460 __u32 xlated_prog_len; 461 462 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 463 perror("bpf_prog_get_info_by_fd failed"); 464 return -1; 465 } 466 467 xlated_prog_len = info.xlated_prog_len; 468 if (xlated_prog_len % buf_element_size) { 469 printf("Program length %u is not multiple of %u\n", 470 xlated_prog_len, buf_element_size); 471 return -1; 472 } 473 474 *cnt = xlated_prog_len / buf_element_size; 475 *buf = calloc(*cnt, buf_element_size); 476 if (!*buf) { 477 perror("can't allocate xlated program buffer"); 478 return -ENOMEM; 479 } 480 481 bzero(&info, sizeof(info)); 482 info.xlated_prog_len = xlated_prog_len; 483 info.xlated_prog_insns = (__u64)(unsigned long)*buf; 484 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 485 perror("second bpf_prog_get_info_by_fd failed"); 486 goto out_free_buf; 487 } 488 489 return 0; 490 491 out_free_buf: 492 free(*buf); 493 *buf = NULL; 494 return -1; 495 } 496 497 bool is_jit_enabled(void) 498 { 499 const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable"; 500 bool enabled = false; 501 int sysctl_fd; 502 503 sysctl_fd = open(jit_sysctl, O_RDONLY); 504 if (sysctl_fd != -1) { 505 char tmpc; 506 507 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1) 508 enabled = (tmpc != '0'); 509 close(sysctl_fd); 510 } 511 512 return enabled; 513 } 514