1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2020 Facebook */ 3 #include <test_progs.h> 4 #include <unistd.h> 5 #include <sys/syscall.h> 6 #include <task_local_storage_helpers.h> 7 #include "bpf_iter_ipv6_route.skel.h" 8 #include "bpf_iter_netlink.skel.h" 9 #include "bpf_iter_bpf_map.skel.h" 10 #include "bpf_iter_tasks.skel.h" 11 #include "bpf_iter_task_stack.skel.h" 12 #include "bpf_iter_task_file.skel.h" 13 #include "bpf_iter_task_vmas.skel.h" 14 #include "bpf_iter_task_btf.skel.h" 15 #include "bpf_iter_tcp4.skel.h" 16 #include "bpf_iter_tcp6.skel.h" 17 #include "bpf_iter_udp4.skel.h" 18 #include "bpf_iter_udp6.skel.h" 19 #include "bpf_iter_unix.skel.h" 20 #include "bpf_iter_vma_offset.skel.h" 21 #include "bpf_iter_test_kern1.skel.h" 22 #include "bpf_iter_test_kern2.skel.h" 23 #include "bpf_iter_test_kern3.skel.h" 24 #include "bpf_iter_test_kern4.skel.h" 25 #include "bpf_iter_bpf_hash_map.skel.h" 26 #include "bpf_iter_bpf_percpu_hash_map.skel.h" 27 #include "bpf_iter_bpf_array_map.skel.h" 28 #include "bpf_iter_bpf_percpu_array_map.skel.h" 29 #include "bpf_iter_bpf_sk_storage_helpers.skel.h" 30 #include "bpf_iter_bpf_sk_storage_map.skel.h" 31 #include "bpf_iter_test_kern5.skel.h" 32 #include "bpf_iter_test_kern6.skel.h" 33 #include "bpf_iter_bpf_link.skel.h" 34 #include "bpf_iter_ksym.skel.h" 35 #include "bpf_iter_sockmap.skel.h" 36 37 static void test_btf_id_or_null(void) 38 { 39 struct bpf_iter_test_kern3 *skel; 40 41 skel = bpf_iter_test_kern3__open_and_load(); 42 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) { 43 bpf_iter_test_kern3__destroy(skel); 44 return; 45 } 46 } 47 48 static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts) 49 { 50 struct bpf_link *link; 51 char buf[16] = {}; 52 int iter_fd, len; 53 54 link = bpf_program__attach_iter(prog, opts); 55 if (!ASSERT_OK_PTR(link, "attach_iter")) 56 return; 57 58 iter_fd = bpf_iter_create(bpf_link__fd(link)); 59 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 60 goto free_link; 61 62 /* not check contents, but ensure read() ends without error */ 63 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 64 ; 65 ASSERT_GE(len, 0, "read"); 66 67 close(iter_fd); 68 69 free_link: 70 bpf_link__destroy(link); 71 } 72 73 static void do_dummy_read(struct bpf_program *prog) 74 { 75 do_dummy_read_opts(prog, NULL); 76 } 77 78 static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog, 79 struct bpf_map *map) 80 { 81 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 82 union bpf_iter_link_info linfo; 83 struct bpf_link *link; 84 char buf[16] = {}; 85 int iter_fd, len; 86 87 memset(&linfo, 0, sizeof(linfo)); 88 linfo.map.map_fd = bpf_map__fd(map); 89 opts.link_info = &linfo; 90 opts.link_info_len = sizeof(linfo); 91 link = bpf_program__attach_iter(prog, &opts); 92 if (!ASSERT_OK_PTR(link, "attach_map_iter")) 93 return; 94 95 iter_fd = bpf_iter_create(bpf_link__fd(link)); 96 if (!ASSERT_GE(iter_fd, 0, "create_map_iter")) { 97 bpf_link__destroy(link); 98 return; 99 } 100 101 /* Close link and map fd prematurely */ 102 bpf_link__destroy(link); 103 bpf_object__destroy_skeleton(*skel); 104 *skel = NULL; 105 106 /* Try to let map free work to run first if map is freed */ 107 usleep(100); 108 /* Memory used by both sock map and sock local storage map are 109 * freed after two synchronize_rcu() calls, so wait for it 110 */ 111 kern_sync_rcu(); 112 kern_sync_rcu(); 113 114 /* Read after both map fd and link fd are closed */ 115 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 116 ; 117 ASSERT_GE(len, 0, "read_iterator"); 118 119 close(iter_fd); 120 } 121 122 static int read_fd_into_buffer(int fd, char *buf, int size) 123 { 124 int bufleft = size; 125 int len; 126 127 do { 128 len = read(fd, buf, bufleft); 129 if (len > 0) { 130 buf += len; 131 bufleft -= len; 132 } 133 } while (len > 0); 134 135 return len < 0 ? len : size - bufleft; 136 } 137 138 static void test_ipv6_route(void) 139 { 140 struct bpf_iter_ipv6_route *skel; 141 142 skel = bpf_iter_ipv6_route__open_and_load(); 143 if (!ASSERT_OK_PTR(skel, "bpf_iter_ipv6_route__open_and_load")) 144 return; 145 146 do_dummy_read(skel->progs.dump_ipv6_route); 147 148 bpf_iter_ipv6_route__destroy(skel); 149 } 150 151 static void test_netlink(void) 152 { 153 struct bpf_iter_netlink *skel; 154 155 skel = bpf_iter_netlink__open_and_load(); 156 if (!ASSERT_OK_PTR(skel, "bpf_iter_netlink__open_and_load")) 157 return; 158 159 do_dummy_read(skel->progs.dump_netlink); 160 161 bpf_iter_netlink__destroy(skel); 162 } 163 164 static void test_bpf_map(void) 165 { 166 struct bpf_iter_bpf_map *skel; 167 168 skel = bpf_iter_bpf_map__open_and_load(); 169 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_map__open_and_load")) 170 return; 171 172 do_dummy_read(skel->progs.dump_bpf_map); 173 174 bpf_iter_bpf_map__destroy(skel); 175 } 176 177 static void check_bpf_link_info(const struct bpf_program *prog) 178 { 179 LIBBPF_OPTS(bpf_iter_attach_opts, opts); 180 union bpf_iter_link_info linfo; 181 struct bpf_link_info info = {}; 182 struct bpf_link *link; 183 __u32 info_len; 184 int err; 185 186 memset(&linfo, 0, sizeof(linfo)); 187 linfo.task.tid = getpid(); 188 opts.link_info = &linfo; 189 opts.link_info_len = sizeof(linfo); 190 191 link = bpf_program__attach_iter(prog, &opts); 192 if (!ASSERT_OK_PTR(link, "attach_iter")) 193 return; 194 195 info_len = sizeof(info); 196 err = bpf_link_get_info_by_fd(bpf_link__fd(link), &info, &info_len); 197 ASSERT_OK(err, "bpf_link_get_info_by_fd"); 198 ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid"); 199 200 bpf_link__destroy(link); 201 } 202 203 static pthread_mutex_t do_nothing_mutex; 204 205 static void *do_nothing_wait(void *arg) 206 { 207 pthread_mutex_lock(&do_nothing_mutex); 208 pthread_mutex_unlock(&do_nothing_mutex); 209 210 pthread_exit(arg); 211 } 212 213 static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts, 214 int *num_unknown, int *num_known) 215 { 216 struct bpf_iter_tasks *skel; 217 pthread_t thread_id; 218 void *ret; 219 220 skel = bpf_iter_tasks__open_and_load(); 221 if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load")) 222 return; 223 224 ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock"); 225 226 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL), 227 "pthread_create"); 228 229 skel->bss->tid = sys_gettid(); 230 231 do_dummy_read_opts(skel->progs.dump_task, opts); 232 233 *num_unknown = skel->bss->num_unknown_tid; 234 *num_known = skel->bss->num_known_tid; 235 236 ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock"); 237 ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL, 238 "pthread_join"); 239 240 bpf_iter_tasks__destroy(skel); 241 } 242 243 static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known) 244 { 245 int num_unknown_tid, num_known_tid; 246 247 test_task_common_nocheck(opts, &num_unknown_tid, &num_known_tid); 248 ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid"); 249 ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid"); 250 } 251 252 static void *run_test_task_tid(void *arg) 253 { 254 LIBBPF_OPTS(bpf_iter_attach_opts, opts); 255 union bpf_iter_link_info linfo; 256 int num_unknown_tid, num_known_tid; 257 258 ASSERT_NEQ(getpid(), sys_gettid(), "check_new_thread_id"); 259 260 memset(&linfo, 0, sizeof(linfo)); 261 linfo.task.tid = sys_gettid(); 262 opts.link_info = &linfo; 263 opts.link_info_len = sizeof(linfo); 264 test_task_common(&opts, 0, 1); 265 266 linfo.task.tid = 0; 267 linfo.task.pid = getpid(); 268 /* This includes the parent thread, this thread, watchdog timer thread 269 * and the do_nothing_wait thread 270 */ 271 test_task_common(&opts, 3, 1); 272 273 test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid); 274 ASSERT_GT(num_unknown_tid, 2, "check_num_unknown_tid"); 275 ASSERT_EQ(num_known_tid, 1, "check_num_known_tid"); 276 277 return NULL; 278 } 279 280 static void test_task_tid(void) 281 { 282 pthread_t thread_id; 283 284 /* Create a new thread so pid and tid aren't the same */ 285 ASSERT_OK(pthread_create(&thread_id, NULL, &run_test_task_tid, NULL), 286 "pthread_create"); 287 ASSERT_FALSE(pthread_join(thread_id, NULL), "pthread_join"); 288 } 289 290 static void test_task_pid(void) 291 { 292 LIBBPF_OPTS(bpf_iter_attach_opts, opts); 293 union bpf_iter_link_info linfo; 294 295 memset(&linfo, 0, sizeof(linfo)); 296 linfo.task.pid = getpid(); 297 opts.link_info = &linfo; 298 opts.link_info_len = sizeof(linfo); 299 300 test_task_common(&opts, 2, 1); 301 } 302 303 static void test_task_pidfd(void) 304 { 305 LIBBPF_OPTS(bpf_iter_attach_opts, opts); 306 union bpf_iter_link_info linfo; 307 int pidfd; 308 309 pidfd = sys_pidfd_open(getpid(), 0); 310 if (!ASSERT_GT(pidfd, 0, "sys_pidfd_open")) 311 return; 312 313 memset(&linfo, 0, sizeof(linfo)); 314 linfo.task.pid_fd = pidfd; 315 opts.link_info = &linfo; 316 opts.link_info_len = sizeof(linfo); 317 318 test_task_common(&opts, 2, 1); 319 320 close(pidfd); 321 } 322 323 static void test_task_sleepable(void) 324 { 325 struct bpf_iter_tasks *skel; 326 int pid, status, err, data_pipe[2], finish_pipe[2], c; 327 char *test_data = NULL; 328 char *test_data_long = NULL; 329 char *data[2]; 330 331 if (!ASSERT_OK(pipe(data_pipe), "data_pipe") || 332 !ASSERT_OK(pipe(finish_pipe), "finish_pipe")) 333 return; 334 335 skel = bpf_iter_tasks__open_and_load(); 336 if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load")) 337 return; 338 339 pid = fork(); 340 if (!ASSERT_GE(pid, 0, "fork")) 341 return; 342 343 if (pid == 0) { 344 /* child */ 345 close(data_pipe[0]); 346 close(finish_pipe[1]); 347 348 test_data = malloc(sizeof(char) * 10); 349 strncpy(test_data, "test_data", 10); 350 test_data[9] = '\0'; 351 352 test_data_long = malloc(sizeof(char) * 5000); 353 for (int i = 0; i < 5000; ++i) { 354 if (i % 2 == 0) 355 test_data_long[i] = 'b'; 356 else 357 test_data_long[i] = 'a'; 358 } 359 test_data_long[4999] = '\0'; 360 361 data[0] = test_data; 362 data[1] = test_data_long; 363 364 write(data_pipe[1], &data, sizeof(data)); 365 366 /* keep child alive until after the test */ 367 err = read(finish_pipe[0], &c, 1); 368 if (err != 1) 369 exit(-1); 370 371 close(data_pipe[1]); 372 close(finish_pipe[0]); 373 _exit(0); 374 } 375 376 /* parent */ 377 close(data_pipe[1]); 378 close(finish_pipe[0]); 379 380 err = read(data_pipe[0], &data, sizeof(data)); 381 ASSERT_EQ(err, sizeof(data), "read_check"); 382 383 skel->bss->user_ptr = data[0]; 384 skel->bss->user_ptr_long = data[1]; 385 skel->bss->pid = pid; 386 387 do_dummy_read(skel->progs.dump_task_sleepable); 388 389 ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0, 390 "num_expected_failure_copy_from_user_task"); 391 ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0, 392 "num_success_copy_from_user_task"); 393 ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task_str, 0, 394 "num_expected_failure_copy_from_user_task_str"); 395 ASSERT_GT(skel->bss->num_success_copy_from_user_task_str, 0, 396 "num_success_copy_from_user_task_str"); 397 398 bpf_iter_tasks__destroy(skel); 399 400 write(finish_pipe[1], &c, 1); 401 err = waitpid(pid, &status, 0); 402 ASSERT_EQ(err, pid, "waitpid"); 403 ASSERT_EQ(status, 0, "zero_child_exit"); 404 405 close(data_pipe[0]); 406 close(finish_pipe[1]); 407 } 408 409 static void test_task_stack(void) 410 { 411 struct bpf_iter_task_stack *skel; 412 413 skel = bpf_iter_task_stack__open_and_load(); 414 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_stack__open_and_load")) 415 return; 416 417 do_dummy_read(skel->progs.dump_task_stack); 418 do_dummy_read(skel->progs.get_task_user_stacks); 419 420 ASSERT_EQ(skel->bss->num_user_stacks, 1, "num_user_stacks"); 421 422 bpf_iter_task_stack__destroy(skel); 423 } 424 425 static void test_task_file(void) 426 { 427 LIBBPF_OPTS(bpf_iter_attach_opts, opts); 428 struct bpf_iter_task_file *skel; 429 union bpf_iter_link_info linfo; 430 pthread_t thread_id; 431 void *ret; 432 433 skel = bpf_iter_task_file__open_and_load(); 434 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_file__open_and_load")) 435 return; 436 437 skel->bss->tgid = getpid(); 438 439 ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock"); 440 441 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL), 442 "pthread_create"); 443 444 memset(&linfo, 0, sizeof(linfo)); 445 linfo.task.tid = getpid(); 446 opts.link_info = &linfo; 447 opts.link_info_len = sizeof(linfo); 448 449 do_dummy_read_opts(skel->progs.dump_task_file, &opts); 450 451 ASSERT_EQ(skel->bss->count, 0, "check_count"); 452 ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count"); 453 454 skel->bss->last_tgid = 0; 455 skel->bss->count = 0; 456 skel->bss->unique_tgid_count = 0; 457 458 do_dummy_read(skel->progs.dump_task_file); 459 460 ASSERT_EQ(skel->bss->count, 0, "check_count"); 461 ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count"); 462 463 check_bpf_link_info(skel->progs.dump_task_file); 464 465 ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock"); 466 ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join"); 467 ASSERT_NULL(ret, "pthread_join"); 468 469 bpf_iter_task_file__destroy(skel); 470 } 471 472 #define TASKBUFSZ 32768 473 474 static char taskbuf[TASKBUFSZ]; 475 476 static int do_btf_read(struct bpf_iter_task_btf *skel) 477 { 478 struct bpf_program *prog = skel->progs.dump_task_struct; 479 struct bpf_iter_task_btf__bss *bss = skel->bss; 480 int iter_fd = -1, err; 481 struct bpf_link *link; 482 char *buf = taskbuf; 483 int ret = 0; 484 485 link = bpf_program__attach_iter(prog, NULL); 486 if (!ASSERT_OK_PTR(link, "attach_iter")) 487 return ret; 488 489 iter_fd = bpf_iter_create(bpf_link__fd(link)); 490 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 491 goto free_link; 492 493 err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ); 494 if (bss->skip) { 495 printf("%s:SKIP:no __builtin_btf_type_id\n", __func__); 496 ret = 1; 497 test__skip(); 498 goto free_link; 499 } 500 501 if (!ASSERT_GE(err, 0, "read")) 502 goto free_link; 503 504 ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)", 505 "check for btf representation of task_struct in iter data"); 506 free_link: 507 if (iter_fd > 0) 508 close(iter_fd); 509 bpf_link__destroy(link); 510 return ret; 511 } 512 513 static void test_task_btf(void) 514 { 515 struct bpf_iter_task_btf__bss *bss; 516 struct bpf_iter_task_btf *skel; 517 int ret; 518 519 skel = bpf_iter_task_btf__open_and_load(); 520 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_btf__open_and_load")) 521 return; 522 523 bss = skel->bss; 524 525 ret = do_btf_read(skel); 526 if (ret) 527 goto cleanup; 528 529 if (!ASSERT_NEQ(bss->tasks, 0, "no task iteration, did BPF program run?")) 530 goto cleanup; 531 532 ASSERT_EQ(bss->seq_err, 0, "check for unexpected err"); 533 534 cleanup: 535 bpf_iter_task_btf__destroy(skel); 536 } 537 538 static void test_tcp4(void) 539 { 540 struct bpf_iter_tcp4 *skel; 541 542 skel = bpf_iter_tcp4__open_and_load(); 543 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp4__open_and_load")) 544 return; 545 546 do_dummy_read(skel->progs.dump_tcp4); 547 548 bpf_iter_tcp4__destroy(skel); 549 } 550 551 static void test_tcp6(void) 552 { 553 struct bpf_iter_tcp6 *skel; 554 555 skel = bpf_iter_tcp6__open_and_load(); 556 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp6__open_and_load")) 557 return; 558 559 do_dummy_read(skel->progs.dump_tcp6); 560 561 bpf_iter_tcp6__destroy(skel); 562 } 563 564 static void test_udp4(void) 565 { 566 struct bpf_iter_udp4 *skel; 567 568 skel = bpf_iter_udp4__open_and_load(); 569 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp4__open_and_load")) 570 return; 571 572 do_dummy_read(skel->progs.dump_udp4); 573 574 bpf_iter_udp4__destroy(skel); 575 } 576 577 static void test_udp6(void) 578 { 579 struct bpf_iter_udp6 *skel; 580 581 skel = bpf_iter_udp6__open_and_load(); 582 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp6__open_and_load")) 583 return; 584 585 do_dummy_read(skel->progs.dump_udp6); 586 587 bpf_iter_udp6__destroy(skel); 588 } 589 590 static void test_unix(void) 591 { 592 struct bpf_iter_unix *skel; 593 594 skel = bpf_iter_unix__open_and_load(); 595 if (!ASSERT_OK_PTR(skel, "bpf_iter_unix__open_and_load")) 596 return; 597 598 do_dummy_read(skel->progs.dump_unix); 599 600 bpf_iter_unix__destroy(skel); 601 } 602 603 /* The expected string is less than 16 bytes */ 604 static int do_read_with_fd(int iter_fd, const char *expected, 605 bool read_one_char) 606 { 607 int len, read_buf_len, start; 608 char buf[16] = {}; 609 610 read_buf_len = read_one_char ? 1 : 16; 611 start = 0; 612 while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) { 613 start += len; 614 if (!ASSERT_LT(start, 16, "read")) 615 return -1; 616 read_buf_len = read_one_char ? 1 : 16 - start; 617 } 618 if (!ASSERT_GE(len, 0, "read")) 619 return -1; 620 621 if (!ASSERT_STREQ(buf, expected, "read")) 622 return -1; 623 624 return 0; 625 } 626 627 static void test_anon_iter(bool read_one_char) 628 { 629 struct bpf_iter_test_kern1 *skel; 630 struct bpf_link *link; 631 int iter_fd, err; 632 633 skel = bpf_iter_test_kern1__open_and_load(); 634 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern1__open_and_load")) 635 return; 636 637 err = bpf_iter_test_kern1__attach(skel); 638 if (!ASSERT_OK(err, "bpf_iter_test_kern1__attach")) { 639 goto out; 640 } 641 642 link = skel->links.dump_task; 643 iter_fd = bpf_iter_create(bpf_link__fd(link)); 644 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 645 goto out; 646 647 do_read_with_fd(iter_fd, "abcd", read_one_char); 648 close(iter_fd); 649 650 out: 651 bpf_iter_test_kern1__destroy(skel); 652 } 653 654 static int do_read(const char *path, const char *expected) 655 { 656 int err, iter_fd; 657 658 iter_fd = open(path, O_RDONLY); 659 if (!ASSERT_GE(iter_fd, 0, "open")) 660 return -1; 661 662 err = do_read_with_fd(iter_fd, expected, false); 663 close(iter_fd); 664 return err; 665 } 666 667 static void test_file_iter(void) 668 { 669 const char *path = "/sys/fs/bpf/bpf_iter_test1"; 670 struct bpf_iter_test_kern1 *skel1; 671 struct bpf_iter_test_kern2 *skel2; 672 struct bpf_link *link; 673 int err; 674 675 skel1 = bpf_iter_test_kern1__open_and_load(); 676 if (!ASSERT_OK_PTR(skel1, "bpf_iter_test_kern1__open_and_load")) 677 return; 678 679 link = bpf_program__attach_iter(skel1->progs.dump_task, NULL); 680 if (!ASSERT_OK_PTR(link, "attach_iter")) 681 goto out; 682 683 /* unlink this path if it exists. */ 684 unlink(path); 685 686 err = bpf_link__pin(link, path); 687 if (!ASSERT_OK(err, "pin_iter")) 688 goto free_link; 689 690 err = do_read(path, "abcd"); 691 if (err) 692 goto unlink_path; 693 694 /* file based iterator seems working fine. Let us a link update 695 * of the underlying link and `cat` the iterator again, its content 696 * should change. 697 */ 698 skel2 = bpf_iter_test_kern2__open_and_load(); 699 if (!ASSERT_OK_PTR(skel2, "bpf_iter_test_kern2__open_and_load")) 700 goto unlink_path; 701 702 err = bpf_link__update_program(link, skel2->progs.dump_task); 703 if (!ASSERT_OK(err, "update_prog")) 704 goto destroy_skel2; 705 706 do_read(path, "ABCD"); 707 708 destroy_skel2: 709 bpf_iter_test_kern2__destroy(skel2); 710 unlink_path: 711 unlink(path); 712 free_link: 713 bpf_link__destroy(link); 714 out: 715 bpf_iter_test_kern1__destroy(skel1); 716 } 717 718 static void test_overflow(bool test_e2big_overflow, bool ret1) 719 { 720 __u32 map_info_len, total_read_len, expected_read_len; 721 int err, iter_fd, map1_fd, map2_fd, len; 722 struct bpf_map_info map_info = {}; 723 struct bpf_iter_test_kern4 *skel; 724 struct bpf_link *link; 725 __u32 iter_size; 726 char *buf; 727 728 skel = bpf_iter_test_kern4__open(); 729 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern4__open")) 730 return; 731 732 /* create two maps: bpf program will only do bpf_seq_write 733 * for these two maps. The goal is one map output almost 734 * fills seq_file buffer and then the other will trigger 735 * overflow and needs restart. 736 */ 737 map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL); 738 if (!ASSERT_GE(map1_fd, 0, "bpf_map_create")) 739 goto out; 740 map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL); 741 if (!ASSERT_GE(map2_fd, 0, "bpf_map_create")) 742 goto free_map1; 743 744 /* bpf_seq_printf kernel buffer is 8 pages, so one map 745 * bpf_seq_write will mostly fill it, and the other map 746 * will partially fill and then trigger overflow and need 747 * bpf_seq_read restart. 748 */ 749 iter_size = sysconf(_SC_PAGE_SIZE) << 3; 750 751 if (test_e2big_overflow) { 752 skel->rodata->print_len = (iter_size + 8) / 8; 753 expected_read_len = 2 * (iter_size + 8); 754 } else if (!ret1) { 755 skel->rodata->print_len = (iter_size - 8) / 8; 756 expected_read_len = 2 * (iter_size - 8); 757 } else { 758 skel->rodata->print_len = 1; 759 expected_read_len = 2 * 8; 760 } 761 skel->rodata->ret1 = ret1; 762 763 if (!ASSERT_OK(bpf_iter_test_kern4__load(skel), 764 "bpf_iter_test_kern4__load")) 765 goto free_map2; 766 767 /* setup filtering map_id in bpf program */ 768 map_info_len = sizeof(map_info); 769 err = bpf_map_get_info_by_fd(map1_fd, &map_info, &map_info_len); 770 if (!ASSERT_OK(err, "get_map_info")) 771 goto free_map2; 772 skel->bss->map1_id = map_info.id; 773 774 err = bpf_map_get_info_by_fd(map2_fd, &map_info, &map_info_len); 775 if (!ASSERT_OK(err, "get_map_info")) 776 goto free_map2; 777 skel->bss->map2_id = map_info.id; 778 779 link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL); 780 if (!ASSERT_OK_PTR(link, "attach_iter")) 781 goto free_map2; 782 783 iter_fd = bpf_iter_create(bpf_link__fd(link)); 784 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 785 goto free_link; 786 787 buf = malloc(expected_read_len); 788 if (!ASSERT_OK_PTR(buf, "malloc")) 789 goto close_iter; 790 791 /* do read */ 792 total_read_len = 0; 793 if (test_e2big_overflow) { 794 while ((len = read(iter_fd, buf, expected_read_len)) > 0) 795 total_read_len += len; 796 797 ASSERT_EQ(len, -1, "read"); 798 ASSERT_EQ(errno, E2BIG, "read"); 799 goto free_buf; 800 } else if (!ret1) { 801 while ((len = read(iter_fd, buf, expected_read_len)) > 0) 802 total_read_len += len; 803 804 if (!ASSERT_GE(len, 0, "read")) 805 goto free_buf; 806 } else { 807 do { 808 len = read(iter_fd, buf, expected_read_len); 809 if (len > 0) 810 total_read_len += len; 811 } while (len > 0 || len == -EAGAIN); 812 813 if (!ASSERT_GE(len, 0, "read")) 814 goto free_buf; 815 } 816 817 if (!ASSERT_EQ(total_read_len, expected_read_len, "read")) 818 goto free_buf; 819 820 if (!ASSERT_EQ(skel->bss->map1_accessed, 1, "map1_accessed")) 821 goto free_buf; 822 823 if (!ASSERT_EQ(skel->bss->map2_accessed, 2, "map2_accessed")) 824 goto free_buf; 825 826 ASSERT_EQ(skel->bss->map2_seqnum1, skel->bss->map2_seqnum2, "map2_seqnum"); 827 828 free_buf: 829 free(buf); 830 close_iter: 831 close(iter_fd); 832 free_link: 833 bpf_link__destroy(link); 834 free_map2: 835 close(map2_fd); 836 free_map1: 837 close(map1_fd); 838 out: 839 bpf_iter_test_kern4__destroy(skel); 840 } 841 842 static void test_bpf_hash_map(void) 843 { 844 __u32 expected_key_a = 0, expected_key_b = 0; 845 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 846 struct bpf_iter_bpf_hash_map *skel; 847 int err, i, len, map_fd, iter_fd; 848 union bpf_iter_link_info linfo; 849 __u64 val, expected_val = 0; 850 struct bpf_link *link; 851 struct key_t { 852 int a; 853 int b; 854 int c; 855 } key; 856 char buf[64]; 857 858 skel = bpf_iter_bpf_hash_map__open(); 859 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_hash_map__open")) 860 return; 861 862 skel->bss->in_test_mode = true; 863 864 err = bpf_iter_bpf_hash_map__load(skel); 865 if (!ASSERT_OK(err, "bpf_iter_bpf_hash_map__load")) 866 goto out; 867 868 /* iterator with hashmap2 and hashmap3 should fail */ 869 memset(&linfo, 0, sizeof(linfo)); 870 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2); 871 opts.link_info = &linfo; 872 opts.link_info_len = sizeof(linfo); 873 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 874 if (!ASSERT_ERR_PTR(link, "attach_iter")) 875 goto out; 876 877 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3); 878 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 879 if (!ASSERT_ERR_PTR(link, "attach_iter")) 880 goto out; 881 882 /* hashmap1 should be good, update map values here */ 883 map_fd = bpf_map__fd(skel->maps.hashmap1); 884 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) { 885 key.a = i + 1; 886 key.b = i + 2; 887 key.c = i + 3; 888 val = i + 4; 889 expected_key_a += key.a; 890 expected_key_b += key.b; 891 expected_val += val; 892 893 err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY); 894 if (!ASSERT_OK(err, "map_update")) 895 goto out; 896 } 897 898 /* Sleepable program is prohibited for hash map iterator */ 899 linfo.map.map_fd = map_fd; 900 link = bpf_program__attach_iter(skel->progs.sleepable_dummy_dump, &opts); 901 if (!ASSERT_ERR_PTR(link, "attach_sleepable_prog_to_iter")) 902 goto out; 903 904 linfo.map.map_fd = map_fd; 905 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 906 if (!ASSERT_OK_PTR(link, "attach_iter")) 907 goto out; 908 909 iter_fd = bpf_iter_create(bpf_link__fd(link)); 910 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 911 goto free_link; 912 913 /* do some tests */ 914 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 915 ; 916 if (!ASSERT_GE(len, 0, "read")) 917 goto close_iter; 918 919 /* test results */ 920 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a")) 921 goto close_iter; 922 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b")) 923 goto close_iter; 924 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum")) 925 goto close_iter; 926 927 close_iter: 928 close(iter_fd); 929 free_link: 930 bpf_link__destroy(link); 931 out: 932 bpf_iter_bpf_hash_map__destroy(skel); 933 } 934 935 static void test_bpf_percpu_hash_map(void) 936 { 937 __u32 expected_key_a = 0, expected_key_b = 0; 938 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 939 struct bpf_iter_bpf_percpu_hash_map *skel; 940 int err, i, j, len, map_fd, iter_fd; 941 union bpf_iter_link_info linfo; 942 __u32 expected_val = 0; 943 struct bpf_link *link; 944 struct key_t { 945 int a; 946 int b; 947 int c; 948 } key; 949 char buf[64]; 950 void *val; 951 952 skel = bpf_iter_bpf_percpu_hash_map__open(); 953 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__open")) 954 return; 955 956 skel->rodata->num_cpus = bpf_num_possible_cpus(); 957 val = malloc(8 * bpf_num_possible_cpus()); 958 if (!ASSERT_OK_PTR(val, "malloc")) 959 goto out; 960 961 err = bpf_iter_bpf_percpu_hash_map__load(skel); 962 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load")) 963 goto out; 964 965 /* update map values here */ 966 map_fd = bpf_map__fd(skel->maps.hashmap1); 967 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) { 968 key.a = i + 1; 969 key.b = i + 2; 970 key.c = i + 3; 971 expected_key_a += key.a; 972 expected_key_b += key.b; 973 974 for (j = 0; j < bpf_num_possible_cpus(); j++) { 975 *(__u32 *)(val + j * 8) = i + j; 976 expected_val += i + j; 977 } 978 979 err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY); 980 if (!ASSERT_OK(err, "map_update")) 981 goto out; 982 } 983 984 memset(&linfo, 0, sizeof(linfo)); 985 linfo.map.map_fd = map_fd; 986 opts.link_info = &linfo; 987 opts.link_info_len = sizeof(linfo); 988 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts); 989 if (!ASSERT_OK_PTR(link, "attach_iter")) 990 goto out; 991 992 iter_fd = bpf_iter_create(bpf_link__fd(link)); 993 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 994 goto free_link; 995 996 /* do some tests */ 997 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 998 ; 999 if (!ASSERT_GE(len, 0, "read")) 1000 goto close_iter; 1001 1002 /* test results */ 1003 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a")) 1004 goto close_iter; 1005 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b")) 1006 goto close_iter; 1007 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum")) 1008 goto close_iter; 1009 1010 close_iter: 1011 close(iter_fd); 1012 free_link: 1013 bpf_link__destroy(link); 1014 out: 1015 bpf_iter_bpf_percpu_hash_map__destroy(skel); 1016 free(val); 1017 } 1018 1019 static void test_bpf_array_map(void) 1020 { 1021 __u64 val, expected_val = 0, res_first_val, first_val = 0; 1022 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 1023 __u32 key, expected_key = 0, res_first_key; 1024 int err, i, map_fd, hash_fd, iter_fd; 1025 struct bpf_iter_bpf_array_map *skel; 1026 union bpf_iter_link_info linfo; 1027 struct bpf_link *link; 1028 char buf[64] = {}; 1029 int len, start; 1030 1031 skel = bpf_iter_bpf_array_map__open_and_load(); 1032 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load")) 1033 return; 1034 1035 map_fd = bpf_map__fd(skel->maps.arraymap1); 1036 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) { 1037 val = i + 4; 1038 expected_key += i; 1039 expected_val += val; 1040 1041 if (i == 0) 1042 first_val = val; 1043 1044 err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY); 1045 if (!ASSERT_OK(err, "map_update")) 1046 goto out; 1047 } 1048 1049 memset(&linfo, 0, sizeof(linfo)); 1050 linfo.map.map_fd = map_fd; 1051 opts.link_info = &linfo; 1052 opts.link_info_len = sizeof(linfo); 1053 link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts); 1054 if (!ASSERT_OK_PTR(link, "attach_iter")) 1055 goto out; 1056 1057 iter_fd = bpf_iter_create(bpf_link__fd(link)); 1058 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 1059 goto free_link; 1060 1061 /* do some tests */ 1062 start = 0; 1063 while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0) 1064 start += len; 1065 if (!ASSERT_GE(len, 0, "read")) 1066 goto close_iter; 1067 1068 /* test results */ 1069 res_first_key = *(__u32 *)buf; 1070 res_first_val = *(__u64 *)(buf + sizeof(__u32)); 1071 if (!ASSERT_EQ(res_first_key, 0, "bpf_seq_write") || 1072 !ASSERT_EQ(res_first_val, first_val, "bpf_seq_write")) 1073 goto close_iter; 1074 1075 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum")) 1076 goto close_iter; 1077 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum")) 1078 goto close_iter; 1079 1080 hash_fd = bpf_map__fd(skel->maps.hashmap1); 1081 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) { 1082 err = bpf_map_lookup_elem(map_fd, &i, &val); 1083 if (!ASSERT_OK(err, "map_lookup arraymap1")) 1084 goto close_iter; 1085 if (!ASSERT_EQ(i, val, "invalid_val arraymap1")) 1086 goto close_iter; 1087 1088 val = i + 4; 1089 err = bpf_map_lookup_elem(hash_fd, &val, &key); 1090 if (!ASSERT_OK(err, "map_lookup hashmap1")) 1091 goto close_iter; 1092 if (!ASSERT_EQ(key, val - 4, "invalid_val hashmap1")) 1093 goto close_iter; 1094 } 1095 1096 close_iter: 1097 close(iter_fd); 1098 free_link: 1099 bpf_link__destroy(link); 1100 out: 1101 bpf_iter_bpf_array_map__destroy(skel); 1102 } 1103 1104 static void test_bpf_array_map_iter_fd(void) 1105 { 1106 struct bpf_iter_bpf_array_map *skel; 1107 1108 skel = bpf_iter_bpf_array_map__open_and_load(); 1109 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load")) 1110 return; 1111 1112 do_read_map_iter_fd(&skel->skeleton, skel->progs.dump_bpf_array_map, 1113 skel->maps.arraymap1); 1114 1115 bpf_iter_bpf_array_map__destroy(skel); 1116 } 1117 1118 static void test_bpf_percpu_array_map(void) 1119 { 1120 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 1121 struct bpf_iter_bpf_percpu_array_map *skel; 1122 __u32 expected_key = 0, expected_val = 0; 1123 union bpf_iter_link_info linfo; 1124 int err, i, j, map_fd, iter_fd; 1125 struct bpf_link *link; 1126 char buf[64]; 1127 void *val; 1128 int len; 1129 1130 skel = bpf_iter_bpf_percpu_array_map__open(); 1131 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__open")) 1132 return; 1133 1134 skel->rodata->num_cpus = bpf_num_possible_cpus(); 1135 val = malloc(8 * bpf_num_possible_cpus()); 1136 if (!ASSERT_OK_PTR(val, "malloc")) 1137 goto out; 1138 1139 err = bpf_iter_bpf_percpu_array_map__load(skel); 1140 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load")) 1141 goto out; 1142 1143 /* update map values here */ 1144 map_fd = bpf_map__fd(skel->maps.arraymap1); 1145 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) { 1146 expected_key += i; 1147 1148 for (j = 0; j < bpf_num_possible_cpus(); j++) { 1149 *(__u32 *)(val + j * 8) = i + j; 1150 expected_val += i + j; 1151 } 1152 1153 err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY); 1154 if (!ASSERT_OK(err, "map_update")) 1155 goto out; 1156 } 1157 1158 memset(&linfo, 0, sizeof(linfo)); 1159 linfo.map.map_fd = map_fd; 1160 opts.link_info = &linfo; 1161 opts.link_info_len = sizeof(linfo); 1162 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts); 1163 if (!ASSERT_OK_PTR(link, "attach_iter")) 1164 goto out; 1165 1166 iter_fd = bpf_iter_create(bpf_link__fd(link)); 1167 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 1168 goto free_link; 1169 1170 /* do some tests */ 1171 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 1172 ; 1173 if (!ASSERT_GE(len, 0, "read")) 1174 goto close_iter; 1175 1176 /* test results */ 1177 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum")) 1178 goto close_iter; 1179 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum")) 1180 goto close_iter; 1181 1182 close_iter: 1183 close(iter_fd); 1184 free_link: 1185 bpf_link__destroy(link); 1186 out: 1187 bpf_iter_bpf_percpu_array_map__destroy(skel); 1188 free(val); 1189 } 1190 1191 /* An iterator program deletes all local storage in a map. */ 1192 static void test_bpf_sk_storage_delete(void) 1193 { 1194 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 1195 struct bpf_iter_bpf_sk_storage_helpers *skel; 1196 union bpf_iter_link_info linfo; 1197 int err, len, map_fd, iter_fd; 1198 struct bpf_link *link; 1199 int sock_fd = -1; 1200 __u32 val = 42; 1201 char buf[64]; 1202 1203 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load(); 1204 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load")) 1205 return; 1206 1207 map_fd = bpf_map__fd(skel->maps.sk_stg_map); 1208 1209 sock_fd = socket(AF_INET6, SOCK_STREAM, 0); 1210 if (!ASSERT_GE(sock_fd, 0, "socket")) 1211 goto out; 1212 1213 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST); 1214 if (!ASSERT_OK(err, "map_update")) 1215 goto out; 1216 1217 memset(&linfo, 0, sizeof(linfo)); 1218 linfo.map.map_fd = map_fd; 1219 opts.link_info = &linfo; 1220 opts.link_info_len = sizeof(linfo); 1221 link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map, 1222 &opts); 1223 if (!ASSERT_OK_PTR(link, "attach_iter")) 1224 goto out; 1225 1226 iter_fd = bpf_iter_create(bpf_link__fd(link)); 1227 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 1228 goto free_link; 1229 1230 /* do some tests */ 1231 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 1232 ; 1233 if (!ASSERT_GE(len, 0, "read")) 1234 goto close_iter; 1235 1236 /* test results */ 1237 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val); 1238 1239 /* Note: The following assertions serve to ensure 1240 * the value was deleted. It does so by asserting 1241 * that bpf_map_lookup_elem has failed. This might 1242 * seem counterintuitive at first. 1243 */ 1244 ASSERT_ERR(err, "bpf_map_lookup_elem"); 1245 ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem"); 1246 1247 close_iter: 1248 close(iter_fd); 1249 free_link: 1250 bpf_link__destroy(link); 1251 out: 1252 if (sock_fd >= 0) 1253 close(sock_fd); 1254 bpf_iter_bpf_sk_storage_helpers__destroy(skel); 1255 } 1256 1257 /* This creates a socket and its local storage. It then runs a task_iter BPF 1258 * program that replaces the existing socket local storage with the tgid of the 1259 * only task owning a file descriptor to this socket, this process, prog_tests. 1260 * It then runs a tcp socket iterator that negates the value in the existing 1261 * socket local storage, the test verifies that the resulting value is -pid. 1262 */ 1263 static void test_bpf_sk_storage_get(void) 1264 { 1265 struct bpf_iter_bpf_sk_storage_helpers *skel; 1266 int err, map_fd, val = -1; 1267 int sock_fd = -1; 1268 1269 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load(); 1270 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load")) 1271 return; 1272 1273 sock_fd = socket(AF_INET6, SOCK_STREAM, 0); 1274 if (!ASSERT_GE(sock_fd, 0, "socket")) 1275 goto out; 1276 1277 err = listen(sock_fd, 1); 1278 if (!ASSERT_OK(err, "listen")) 1279 goto close_socket; 1280 1281 map_fd = bpf_map__fd(skel->maps.sk_stg_map); 1282 1283 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST); 1284 if (!ASSERT_OK(err, "bpf_map_update_elem")) 1285 goto close_socket; 1286 1287 do_dummy_read(skel->progs.fill_socket_owner); 1288 1289 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val); 1290 if (!ASSERT_OK(err, "bpf_map_lookup_elem") || 1291 !ASSERT_EQ(val, getpid(), "bpf_map_lookup_elem")) 1292 goto close_socket; 1293 1294 do_dummy_read(skel->progs.negate_socket_local_storage); 1295 1296 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val); 1297 ASSERT_OK(err, "bpf_map_lookup_elem"); 1298 ASSERT_EQ(val, -getpid(), "bpf_map_lookup_elem"); 1299 1300 close_socket: 1301 close(sock_fd); 1302 out: 1303 bpf_iter_bpf_sk_storage_helpers__destroy(skel); 1304 } 1305 1306 static void test_bpf_sk_storage_map_iter_fd(void) 1307 { 1308 struct bpf_iter_bpf_sk_storage_map *skel; 1309 1310 skel = bpf_iter_bpf_sk_storage_map__open_and_load(); 1311 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load")) 1312 return; 1313 1314 do_read_map_iter_fd(&skel->skeleton, skel->progs.rw_bpf_sk_storage_map, 1315 skel->maps.sk_stg_map); 1316 1317 bpf_iter_bpf_sk_storage_map__destroy(skel); 1318 } 1319 1320 static void test_bpf_sk_storage_map(void) 1321 { 1322 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 1323 int err, i, len, map_fd, iter_fd, num_sockets; 1324 struct bpf_iter_bpf_sk_storage_map *skel; 1325 union bpf_iter_link_info linfo; 1326 int sock_fd[3] = {-1, -1, -1}; 1327 __u32 val, expected_val = 0; 1328 struct bpf_link *link; 1329 char buf[64]; 1330 1331 skel = bpf_iter_bpf_sk_storage_map__open_and_load(); 1332 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load")) 1333 return; 1334 1335 map_fd = bpf_map__fd(skel->maps.sk_stg_map); 1336 num_sockets = ARRAY_SIZE(sock_fd); 1337 for (i = 0; i < num_sockets; i++) { 1338 sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0); 1339 if (!ASSERT_GE(sock_fd[i], 0, "socket")) 1340 goto out; 1341 1342 val = i + 1; 1343 expected_val += val; 1344 1345 err = bpf_map_update_elem(map_fd, &sock_fd[i], &val, 1346 BPF_NOEXIST); 1347 if (!ASSERT_OK(err, "map_update")) 1348 goto out; 1349 } 1350 1351 memset(&linfo, 0, sizeof(linfo)); 1352 linfo.map.map_fd = map_fd; 1353 opts.link_info = &linfo; 1354 opts.link_info_len = sizeof(linfo); 1355 link = bpf_program__attach_iter(skel->progs.oob_write_bpf_sk_storage_map, &opts); 1356 err = libbpf_get_error(link); 1357 if (!ASSERT_EQ(err, -EACCES, "attach_oob_write_iter")) { 1358 if (!err) 1359 bpf_link__destroy(link); 1360 goto out; 1361 } 1362 1363 link = bpf_program__attach_iter(skel->progs.rw_bpf_sk_storage_map, &opts); 1364 if (!ASSERT_OK_PTR(link, "attach_iter")) 1365 goto out; 1366 1367 iter_fd = bpf_iter_create(bpf_link__fd(link)); 1368 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 1369 goto free_link; 1370 1371 skel->bss->to_add_val = time(NULL); 1372 /* do some tests */ 1373 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 1374 ; 1375 if (!ASSERT_GE(len, 0, "read")) 1376 goto close_iter; 1377 1378 /* test results */ 1379 if (!ASSERT_EQ(skel->bss->ipv6_sk_count, num_sockets, "ipv6_sk_count")) 1380 goto close_iter; 1381 1382 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum")) 1383 goto close_iter; 1384 1385 for (i = 0; i < num_sockets; i++) { 1386 err = bpf_map_lookup_elem(map_fd, &sock_fd[i], &val); 1387 if (!ASSERT_OK(err, "map_lookup") || 1388 !ASSERT_EQ(val, i + 1 + skel->bss->to_add_val, "check_map_value")) 1389 break; 1390 } 1391 1392 close_iter: 1393 close(iter_fd); 1394 free_link: 1395 bpf_link__destroy(link); 1396 out: 1397 for (i = 0; i < num_sockets; i++) { 1398 if (sock_fd[i] >= 0) 1399 close(sock_fd[i]); 1400 } 1401 bpf_iter_bpf_sk_storage_map__destroy(skel); 1402 } 1403 1404 static void test_rdonly_buf_out_of_bound(void) 1405 { 1406 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 1407 struct bpf_iter_test_kern5 *skel; 1408 union bpf_iter_link_info linfo; 1409 struct bpf_link *link; 1410 1411 skel = bpf_iter_test_kern5__open_and_load(); 1412 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern5__open_and_load")) 1413 return; 1414 1415 memset(&linfo, 0, sizeof(linfo)); 1416 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1); 1417 opts.link_info = &linfo; 1418 opts.link_info_len = sizeof(linfo); 1419 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 1420 if (!ASSERT_ERR_PTR(link, "attach_iter")) 1421 bpf_link__destroy(link); 1422 1423 bpf_iter_test_kern5__destroy(skel); 1424 } 1425 1426 static void test_buf_neg_offset(void) 1427 { 1428 struct bpf_iter_test_kern6 *skel; 1429 1430 skel = bpf_iter_test_kern6__open_and_load(); 1431 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern6__open_and_load")) 1432 bpf_iter_test_kern6__destroy(skel); 1433 } 1434 1435 static void test_link_iter(void) 1436 { 1437 struct bpf_iter_bpf_link *skel; 1438 1439 skel = bpf_iter_bpf_link__open_and_load(); 1440 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_link__open_and_load")) 1441 return; 1442 1443 do_dummy_read(skel->progs.dump_bpf_link); 1444 1445 bpf_iter_bpf_link__destroy(skel); 1446 } 1447 1448 static void test_ksym_iter(void) 1449 { 1450 struct bpf_iter_ksym *skel; 1451 1452 skel = bpf_iter_ksym__open_and_load(); 1453 if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load")) 1454 return; 1455 1456 do_dummy_read(skel->progs.dump_ksym); 1457 1458 bpf_iter_ksym__destroy(skel); 1459 } 1460 1461 #define CMP_BUFFER_SIZE 1024 1462 static char task_vma_output[CMP_BUFFER_SIZE]; 1463 static char proc_maps_output[CMP_BUFFER_SIZE]; 1464 1465 /* remove \0 and \t from str, and only keep the first line */ 1466 static void str_strip_first_line(char *str) 1467 { 1468 char *dst = str, *src = str; 1469 1470 do { 1471 if (*src == ' ' || *src == '\t') 1472 src++; 1473 else 1474 *(dst++) = *(src++); 1475 1476 } while (*src != '\0' && *src != '\n'); 1477 1478 *dst = '\0'; 1479 } 1480 1481 static void test_task_vma_common(struct bpf_iter_attach_opts *opts) 1482 { 1483 int err, iter_fd = -1, proc_maps_fd = -1; 1484 struct bpf_iter_task_vmas *skel; 1485 int len, read_size = 4; 1486 char maps_path[64]; 1487 1488 skel = bpf_iter_task_vmas__open(); 1489 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open")) 1490 return; 1491 1492 skel->bss->pid = getpid(); 1493 skel->bss->one_task = opts ? 1 : 0; 1494 1495 err = bpf_iter_task_vmas__load(skel); 1496 if (!ASSERT_OK(err, "bpf_iter_task_vmas__load")) 1497 goto out; 1498 1499 skel->links.proc_maps = bpf_program__attach_iter( 1500 skel->progs.proc_maps, opts); 1501 1502 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) { 1503 skel->links.proc_maps = NULL; 1504 goto out; 1505 } 1506 1507 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps)); 1508 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 1509 goto out; 1510 1511 /* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks 1512 * to trigger seq_file corner cases. 1513 */ 1514 len = 0; 1515 while (len < CMP_BUFFER_SIZE) { 1516 err = read_fd_into_buffer(iter_fd, task_vma_output + len, 1517 MIN(read_size, CMP_BUFFER_SIZE - len)); 1518 if (!err) 1519 break; 1520 if (!ASSERT_GE(err, 0, "read_iter_fd")) 1521 goto out; 1522 len += err; 1523 } 1524 if (opts) 1525 ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task"); 1526 1527 /* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */ 1528 snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid); 1529 proc_maps_fd = open(maps_path, O_RDONLY); 1530 if (!ASSERT_GE(proc_maps_fd, 0, "open_proc_maps")) 1531 goto out; 1532 err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE); 1533 if (!ASSERT_GE(err, 0, "read_prog_maps_fd")) 1534 goto out; 1535 1536 /* strip and compare the first line of the two files */ 1537 str_strip_first_line(task_vma_output); 1538 str_strip_first_line(proc_maps_output); 1539 1540 ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output"); 1541 1542 check_bpf_link_info(skel->progs.proc_maps); 1543 1544 out: 1545 close(proc_maps_fd); 1546 close(iter_fd); 1547 bpf_iter_task_vmas__destroy(skel); 1548 } 1549 1550 static void test_task_vma_dead_task(void) 1551 { 1552 struct bpf_iter_task_vmas *skel; 1553 int wstatus, child_pid = -1; 1554 time_t start_tm, cur_tm; 1555 int err, iter_fd = -1; 1556 int wait_sec = 3; 1557 1558 skel = bpf_iter_task_vmas__open(); 1559 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open")) 1560 return; 1561 1562 skel->bss->pid = getpid(); 1563 1564 err = bpf_iter_task_vmas__load(skel); 1565 if (!ASSERT_OK(err, "bpf_iter_task_vmas__load")) 1566 goto out; 1567 1568 skel->links.proc_maps = bpf_program__attach_iter( 1569 skel->progs.proc_maps, NULL); 1570 1571 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) { 1572 skel->links.proc_maps = NULL; 1573 goto out; 1574 } 1575 1576 start_tm = time(NULL); 1577 cur_tm = start_tm; 1578 1579 child_pid = fork(); 1580 if (child_pid == 0) { 1581 /* Fork short-lived processes in the background. */ 1582 while (cur_tm < start_tm + wait_sec) { 1583 system("echo > /dev/null"); 1584 cur_tm = time(NULL); 1585 } 1586 exit(0); 1587 } 1588 1589 if (!ASSERT_GE(child_pid, 0, "fork_child")) 1590 goto out; 1591 1592 while (cur_tm < start_tm + wait_sec) { 1593 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps)); 1594 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 1595 goto out; 1596 1597 /* Drain all data from iter_fd. */ 1598 while (cur_tm < start_tm + wait_sec) { 1599 err = read_fd_into_buffer(iter_fd, task_vma_output, CMP_BUFFER_SIZE); 1600 if (!ASSERT_GE(err, 0, "read_iter_fd")) 1601 goto out; 1602 1603 cur_tm = time(NULL); 1604 1605 if (err == 0) 1606 break; 1607 } 1608 1609 close(iter_fd); 1610 iter_fd = -1; 1611 } 1612 1613 check_bpf_link_info(skel->progs.proc_maps); 1614 1615 out: 1616 waitpid(child_pid, &wstatus, 0); 1617 close(iter_fd); 1618 bpf_iter_task_vmas__destroy(skel); 1619 } 1620 1621 void test_bpf_sockmap_map_iter_fd(void) 1622 { 1623 struct bpf_iter_sockmap *skel; 1624 1625 skel = bpf_iter_sockmap__open_and_load(); 1626 if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load")) 1627 return; 1628 1629 do_read_map_iter_fd(&skel->skeleton, skel->progs.copy, skel->maps.sockmap); 1630 1631 bpf_iter_sockmap__destroy(skel); 1632 } 1633 1634 static void test_task_vma(void) 1635 { 1636 LIBBPF_OPTS(bpf_iter_attach_opts, opts); 1637 union bpf_iter_link_info linfo; 1638 1639 memset(&linfo, 0, sizeof(linfo)); 1640 linfo.task.tid = getpid(); 1641 opts.link_info = &linfo; 1642 opts.link_info_len = sizeof(linfo); 1643 1644 test_task_vma_common(&opts); 1645 test_task_vma_common(NULL); 1646 } 1647 1648 /* uprobe attach point */ 1649 static noinline int trigger_func(int arg) 1650 { 1651 asm volatile (""); 1652 return arg + 1; 1653 } 1654 1655 static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc) 1656 { 1657 struct bpf_iter_vma_offset *skel; 1658 char buf[16] = {}; 1659 int iter_fd, len; 1660 int pgsz, shift; 1661 1662 skel = bpf_iter_vma_offset__open_and_load(); 1663 if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load")) 1664 return; 1665 1666 skel->bss->pid = getpid(); 1667 skel->bss->address = (uintptr_t)trigger_func; 1668 for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++) 1669 ; 1670 skel->bss->page_shift = shift; 1671 1672 skel->links.get_vma_offset = bpf_program__attach_iter(skel->progs.get_vma_offset, opts); 1673 if (!ASSERT_OK_PTR(skel->links.get_vma_offset, "attach_iter")) 1674 goto exit; 1675 1676 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.get_vma_offset)); 1677 if (!ASSERT_GT(iter_fd, 0, "create_iter")) 1678 goto exit; 1679 1680 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 1681 ; 1682 buf[15] = 0; 1683 ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp"); 1684 1685 ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset"); 1686 if (one_proc) 1687 ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count"); 1688 else 1689 ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count"); 1690 1691 close(iter_fd); 1692 1693 exit: 1694 bpf_iter_vma_offset__destroy(skel); 1695 } 1696 1697 static void test_task_vma_offset(void) 1698 { 1699 LIBBPF_OPTS(bpf_iter_attach_opts, opts); 1700 union bpf_iter_link_info linfo; 1701 1702 memset(&linfo, 0, sizeof(linfo)); 1703 linfo.task.pid = getpid(); 1704 opts.link_info = &linfo; 1705 opts.link_info_len = sizeof(linfo); 1706 1707 test_task_vma_offset_common(&opts, true); 1708 1709 linfo.task.pid = 0; 1710 linfo.task.tid = getpid(); 1711 test_task_vma_offset_common(&opts, true); 1712 1713 test_task_vma_offset_common(NULL, false); 1714 } 1715 1716 void test_bpf_iter(void) 1717 { 1718 ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init"); 1719 1720 if (test__start_subtest("btf_id_or_null")) 1721 test_btf_id_or_null(); 1722 if (test__start_subtest("ipv6_route")) 1723 test_ipv6_route(); 1724 if (test__start_subtest("netlink")) 1725 test_netlink(); 1726 if (test__start_subtest("bpf_map")) 1727 test_bpf_map(); 1728 if (test__start_subtest("task_tid")) 1729 test_task_tid(); 1730 if (test__start_subtest("task_pid")) 1731 test_task_pid(); 1732 if (test__start_subtest("task_pidfd")) 1733 test_task_pidfd(); 1734 if (test__start_subtest("task_sleepable")) 1735 test_task_sleepable(); 1736 if (test__start_subtest("task_stack")) 1737 test_task_stack(); 1738 if (test__start_subtest("task_file")) 1739 test_task_file(); 1740 if (test__start_subtest("task_vma")) 1741 test_task_vma(); 1742 if (test__start_subtest("task_vma_dead_task")) 1743 test_task_vma_dead_task(); 1744 if (test__start_subtest("task_btf")) 1745 test_task_btf(); 1746 if (test__start_subtest("tcp4")) 1747 test_tcp4(); 1748 if (test__start_subtest("tcp6")) 1749 test_tcp6(); 1750 if (test__start_subtest("udp4")) 1751 test_udp4(); 1752 if (test__start_subtest("udp6")) 1753 test_udp6(); 1754 if (test__start_subtest("unix")) 1755 test_unix(); 1756 if (test__start_subtest("anon")) 1757 test_anon_iter(false); 1758 if (test__start_subtest("anon-read-one-char")) 1759 test_anon_iter(true); 1760 if (test__start_subtest("file")) 1761 test_file_iter(); 1762 if (test__start_subtest("overflow")) 1763 test_overflow(false, false); 1764 if (test__start_subtest("overflow-e2big")) 1765 test_overflow(true, false); 1766 if (test__start_subtest("prog-ret-1")) 1767 test_overflow(false, true); 1768 if (test__start_subtest("bpf_hash_map")) 1769 test_bpf_hash_map(); 1770 if (test__start_subtest("bpf_percpu_hash_map")) 1771 test_bpf_percpu_hash_map(); 1772 if (test__start_subtest("bpf_array_map")) 1773 test_bpf_array_map(); 1774 if (test__start_subtest("bpf_array_map_iter_fd")) 1775 test_bpf_array_map_iter_fd(); 1776 if (test__start_subtest("bpf_percpu_array_map")) 1777 test_bpf_percpu_array_map(); 1778 if (test__start_subtest("bpf_sk_storage_map")) 1779 test_bpf_sk_storage_map(); 1780 if (test__start_subtest("bpf_sk_storage_map_iter_fd")) 1781 test_bpf_sk_storage_map_iter_fd(); 1782 if (test__start_subtest("bpf_sk_storage_delete")) 1783 test_bpf_sk_storage_delete(); 1784 if (test__start_subtest("bpf_sk_storage_get")) 1785 test_bpf_sk_storage_get(); 1786 if (test__start_subtest("rdonly-buf-out-of-bound")) 1787 test_rdonly_buf_out_of_bound(); 1788 if (test__start_subtest("buf-neg-offset")) 1789 test_buf_neg_offset(); 1790 if (test__start_subtest("link-iter")) 1791 test_link_iter(); 1792 if (test__start_subtest("ksym")) 1793 test_ksym_iter(); 1794 if (test__start_subtest("bpf_sockmap_map_iter_fd")) 1795 test_bpf_sockmap_map_iter_fd(); 1796 if (test__start_subtest("vma_offset")) 1797 test_task_vma_offset(); 1798 } 1799