1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (C) 2019 Netronome Systems, Inc. */ 3 /* Copyright (C) 2020 Facebook, Inc. */ 4 #include <ctype.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <errno.h> 8 #include <bpf/bpf.h> 9 #include <bpf/libbpf.h> 10 #include "disasm.h" 11 #include "test_progs.h" 12 #include "testing_helpers.h" 13 #include <linux/membarrier.h> 14 15 int parse_num_list(const char *s, bool **num_set, int *num_set_len) 16 { 17 int i, set_len = 0, new_len, num, start = 0, end = -1; 18 bool *set = NULL, *tmp, parsing_end = false; 19 char *next; 20 21 while (s[0]) { 22 errno = 0; 23 num = strtol(s, &next, 10); 24 if (errno) 25 return -errno; 26 27 if (parsing_end) 28 end = num; 29 else 30 start = num; 31 32 if (!parsing_end && *next == '-') { 33 s = next + 1; 34 parsing_end = true; 35 continue; 36 } else if (*next == ',') { 37 parsing_end = false; 38 s = next + 1; 39 end = num; 40 } else if (*next == '\0') { 41 parsing_end = false; 42 s = next; 43 end = num; 44 } else { 45 return -EINVAL; 46 } 47 48 if (start > end) 49 return -EINVAL; 50 51 if (end + 1 > set_len) { 52 new_len = end + 1; 53 tmp = realloc(set, new_len); 54 if (!tmp) { 55 free(set); 56 return -ENOMEM; 57 } 58 for (i = set_len; i < start; i++) 59 tmp[i] = false; 60 set = tmp; 61 set_len = new_len; 62 } 63 for (i = start; i <= end; i++) 64 set[i] = true; 65 } 66 67 if (!set || parsing_end) 68 return -EINVAL; 69 70 *num_set = set; 71 *num_set_len = set_len; 72 73 return 0; 74 } 75 76 static int do_insert_test(struct test_filter_set *set, 77 char *test_str, 78 char *subtest_str) 79 { 80 struct test_filter *tmp, *test; 81 char **ctmp; 82 int i; 83 84 for (i = 0; i < set->cnt; i++) { 85 test = &set->tests[i]; 86 87 if (strcmp(test_str, test->name) == 0) { 88 free(test_str); 89 goto subtest; 90 } 91 } 92 93 tmp = realloc(set->tests, sizeof(*test) * (set->cnt + 1)); 94 if (!tmp) 95 return -ENOMEM; 96 97 set->tests = tmp; 98 test = &set->tests[set->cnt]; 99 100 test->name = test_str; 101 test->subtests = NULL; 102 test->subtest_cnt = 0; 103 104 set->cnt++; 105 106 subtest: 107 if (!subtest_str) 108 return 0; 109 110 for (i = 0; i < test->subtest_cnt; i++) { 111 if (strcmp(subtest_str, test->subtests[i]) == 0) { 112 free(subtest_str); 113 return 0; 114 } 115 } 116 117 ctmp = realloc(test->subtests, 118 sizeof(*test->subtests) * (test->subtest_cnt + 1)); 119 if (!ctmp) 120 return -ENOMEM; 121 122 test->subtests = ctmp; 123 test->subtests[test->subtest_cnt] = subtest_str; 124 125 test->subtest_cnt++; 126 127 return 0; 128 } 129 130 static int insert_test(struct test_filter_set *set, 131 char *test_spec, 132 bool is_glob_pattern) 133 { 134 char *pattern, *subtest_str, *ext_test_str, *ext_subtest_str = NULL; 135 int glob_chars = 0; 136 137 if (is_glob_pattern) { 138 pattern = "%s"; 139 } else { 140 pattern = "*%s*"; 141 glob_chars = 2; 142 } 143 144 subtest_str = strchr(test_spec, '/'); 145 if (subtest_str) { 146 *subtest_str = '\0'; 147 subtest_str += 1; 148 } 149 150 ext_test_str = malloc(strlen(test_spec) + glob_chars + 1); 151 if (!ext_test_str) 152 goto err; 153 154 sprintf(ext_test_str, pattern, test_spec); 155 156 if (subtest_str) { 157 ext_subtest_str = malloc(strlen(subtest_str) + glob_chars + 1); 158 if (!ext_subtest_str) 159 goto err; 160 161 sprintf(ext_subtest_str, pattern, subtest_str); 162 } 163 164 return do_insert_test(set, ext_test_str, ext_subtest_str); 165 166 err: 167 free(ext_test_str); 168 free(ext_subtest_str); 169 170 return -ENOMEM; 171 } 172 173 int parse_test_list_file(const char *path, 174 struct test_filter_set *set, 175 bool is_glob_pattern) 176 { 177 char *buf = NULL, *capture_start, *capture_end, *scan_end; 178 size_t buflen = 0; 179 int err = 0; 180 FILE *f; 181 182 f = fopen(path, "r"); 183 if (!f) { 184 err = -errno; 185 fprintf(stderr, "Failed to open '%s': %d\n", path, err); 186 return err; 187 } 188 189 while (getline(&buf, &buflen, f) != -1) { 190 capture_start = buf; 191 192 while (isspace(*capture_start)) 193 ++capture_start; 194 195 capture_end = capture_start; 196 scan_end = capture_start; 197 198 while (*scan_end && *scan_end != '#') { 199 if (!isspace(*scan_end)) 200 capture_end = scan_end; 201 202 ++scan_end; 203 } 204 205 if (capture_end == capture_start) 206 continue; 207 208 *(++capture_end) = '\0'; 209 210 err = insert_test(set, capture_start, is_glob_pattern); 211 if (err) 212 break; 213 } 214 215 free(buf); 216 fclose(f); 217 return err; 218 } 219 220 int parse_test_list(const char *s, 221 struct test_filter_set *set, 222 bool is_glob_pattern) 223 { 224 char *input, *state = NULL, *test_spec; 225 int err = 0, cnt = 0; 226 227 input = strdup(s); 228 if (!input) 229 return -ENOMEM; 230 231 while ((test_spec = strtok_r(cnt++ ? NULL : input, ",", &state))) { 232 err = insert_test(set, test_spec, is_glob_pattern); 233 if (err) 234 break; 235 } 236 237 free(input); 238 return err; 239 } 240 241 __u32 link_info_prog_id(const struct bpf_link *link, struct bpf_link_info *info) 242 { 243 __u32 info_len = sizeof(*info); 244 int err; 245 246 memset(info, 0, sizeof(*info)); 247 err = bpf_link_get_info_by_fd(bpf_link__fd(link), info, &info_len); 248 if (err) { 249 printf("failed to get link info: %d\n", -errno); 250 return 0; 251 } 252 return info->prog_id; 253 } 254 255 int extra_prog_load_log_flags = 0; 256 257 int testing_prog_flags(void) 258 { 259 static int cached_flags = -1; 260 static int prog_flags[] = { BPF_F_TEST_RND_HI32, BPF_F_TEST_REG_INVARIANTS }; 261 static struct bpf_insn insns[] = { 262 BPF_MOV64_IMM(BPF_REG_0, 0), 263 BPF_EXIT_INSN(), 264 }; 265 int insn_cnt = ARRAY_SIZE(insns), i, fd, flags = 0; 266 LIBBPF_OPTS(bpf_prog_load_opts, opts); 267 268 if (cached_flags >= 0) 269 return cached_flags; 270 271 for (i = 0; i < ARRAY_SIZE(prog_flags); i++) { 272 opts.prog_flags = prog_flags[i]; 273 fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "flag-test", "GPL", 274 insns, insn_cnt, &opts); 275 if (fd >= 0) { 276 flags |= prog_flags[i]; 277 close(fd); 278 } 279 } 280 281 cached_flags = flags; 282 return cached_flags; 283 } 284 285 int bpf_prog_test_load(const char *file, enum bpf_prog_type type, 286 struct bpf_object **pobj, int *prog_fd) 287 { 288 LIBBPF_OPTS(bpf_object_open_opts, opts, 289 .kernel_log_level = extra_prog_load_log_flags, 290 ); 291 struct bpf_object *obj; 292 struct bpf_program *prog; 293 __u32 flags; 294 int err; 295 296 obj = bpf_object__open_file(file, &opts); 297 if (!obj) 298 return -errno; 299 300 prog = bpf_object__next_program(obj, NULL); 301 if (!prog) { 302 err = -ENOENT; 303 goto err_out; 304 } 305 306 if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type) 307 bpf_program__set_type(prog, type); 308 309 flags = bpf_program__flags(prog) | testing_prog_flags(); 310 bpf_program__set_flags(prog, flags); 311 312 err = bpf_object__load(obj); 313 if (err) 314 goto err_out; 315 316 *pobj = obj; 317 *prog_fd = bpf_program__fd(prog); 318 319 return 0; 320 err_out: 321 bpf_object__close(obj); 322 return err; 323 } 324 325 int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, 326 size_t insns_cnt, const char *license, 327 __u32 kern_version, char *log_buf, 328 size_t log_buf_sz) 329 { 330 LIBBPF_OPTS(bpf_prog_load_opts, opts, 331 .kern_version = kern_version, 332 .prog_flags = testing_prog_flags(), 333 .log_level = extra_prog_load_log_flags, 334 .log_buf = log_buf, 335 .log_size = log_buf_sz, 336 ); 337 338 return bpf_prog_load(type, NULL, license, insns, insns_cnt, &opts); 339 } 340 341 __u64 read_perf_max_sample_freq(void) 342 { 343 __u64 sample_freq = 5000; /* fallback to 5000 on error */ 344 FILE *f; 345 346 f = fopen("/proc/sys/kernel/perf_event_max_sample_rate", "r"); 347 if (f == NULL) { 348 printf("Failed to open /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 349 "return default value: 5000\n", -errno); 350 return sample_freq; 351 } 352 if (fscanf(f, "%llu", &sample_freq) != 1) { 353 printf("Failed to parse /proc/sys/kernel/perf_event_max_sample_rate: err %d\n" 354 "return default value: 5000\n", -errno); 355 } 356 357 fclose(f); 358 return sample_freq; 359 } 360 361 int finit_module(int fd, const char *param_values, int flags) 362 { 363 return syscall(__NR_finit_module, fd, param_values, flags); 364 } 365 366 int delete_module(const char *name, int flags) 367 { 368 return syscall(__NR_delete_module, name, flags); 369 } 370 371 int try_unload_module(const char *name, int retries, bool verbose) 372 { 373 int ret, cnt = 0; 374 375 if (kern_sync_rcu()) 376 fprintf(stdout, "Failed to trigger kernel-side RCU sync!\n"); 377 378 for (;;) { 379 ret = delete_module(name, 0); 380 if (!ret || errno != EAGAIN) 381 break; 382 if (++cnt > retries) { 383 fprintf(stdout, "Unload of %s timed out\n", name); 384 break; 385 } 386 usleep(100); 387 } 388 389 if (ret) { 390 if (errno == ENOENT) { 391 if (verbose) 392 fprintf(stdout, "%s.ko is already unloaded.\n", name); 393 return -1; 394 } 395 fprintf(stdout, "Failed to unload %s.ko from kernel: %d\n", name, -errno); 396 return -1; 397 } 398 if (verbose) 399 fprintf(stdout, "Successfully unloaded %s.ko.\n", name); 400 return 0; 401 } 402 403 int unload_module(const char *name, bool verbose) 404 { 405 return try_unload_module(name, 10000, verbose); 406 } 407 408 static int __load_module(const char *path, const char *param_values, bool verbose) 409 { 410 int fd; 411 412 if (verbose) 413 fprintf(stdout, "Loading %s...\n", path); 414 415 fd = open(path, O_RDONLY); 416 if (fd < 0) { 417 fprintf(stdout, "Can't find %s kernel module: %d\n", path, -errno); 418 return -ENOENT; 419 } 420 if (finit_module(fd, param_values, 0)) { 421 fprintf(stdout, "Failed to load %s into the kernel: %d\n", path, -errno); 422 close(fd); 423 return -EINVAL; 424 } 425 close(fd); 426 427 if (verbose) 428 fprintf(stdout, "Successfully loaded %s.\n", path); 429 return 0; 430 } 431 432 int load_module_params(const char *path, const char *param_values, bool verbose) 433 { 434 return __load_module(path, param_values, verbose); 435 } 436 437 int load_module(const char *path, bool verbose) 438 { 439 return __load_module(path, "", verbose); 440 } 441 442 int unload_bpf_testmod(bool verbose) 443 { 444 return unload_module("bpf_testmod", verbose); 445 } 446 447 int load_bpf_testmod(bool verbose) 448 { 449 return load_module("bpf_testmod.ko", verbose); 450 } 451 452 /* 453 * Trigger synchronize_rcu() in kernel. 454 */ 455 int kern_sync_rcu(void) 456 { 457 return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, 0, 0); 458 } 459 460 int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt) 461 { 462 __u32 buf_element_size = sizeof(struct bpf_insn); 463 struct bpf_prog_info info = {}; 464 __u32 info_len = sizeof(info); 465 __u32 xlated_prog_len; 466 467 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 468 perror("bpf_prog_get_info_by_fd failed"); 469 return -1; 470 } 471 472 xlated_prog_len = info.xlated_prog_len; 473 if (xlated_prog_len % buf_element_size) { 474 printf("Program length %u is not multiple of %u\n", 475 xlated_prog_len, buf_element_size); 476 return -1; 477 } 478 479 *cnt = xlated_prog_len / buf_element_size; 480 *buf = calloc(*cnt, buf_element_size); 481 if (!*buf) { 482 perror("can't allocate xlated program buffer"); 483 return -ENOMEM; 484 } 485 486 bzero(&info, sizeof(info)); 487 info.xlated_prog_len = xlated_prog_len; 488 info.xlated_prog_insns = (__u64)(unsigned long)*buf; 489 if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) { 490 perror("second bpf_prog_get_info_by_fd failed"); 491 goto out_free_buf; 492 } 493 494 return 0; 495 496 out_free_buf: 497 free(*buf); 498 *buf = NULL; 499 return -1; 500 } 501 502 bool is_jit_enabled(void) 503 { 504 const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable"; 505 bool enabled = false; 506 int sysctl_fd; 507 508 sysctl_fd = open(jit_sysctl, O_RDONLY); 509 if (sysctl_fd != -1) { 510 char tmpc; 511 512 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1) 513 enabled = (tmpc != '0'); 514 close(sysctl_fd); 515 } 516 517 return enabled; 518 } 519