1 // SPDX-License-Identifier: GPL-2.0 2 #include <unistd.h> 3 #include <test_progs.h> 4 #include <network_helpers.h> 5 #include "tailcall_poke.skel.h" 6 #include "tailcall_bpf2bpf_hierarchy2.skel.h" 7 #include "tailcall_bpf2bpf_hierarchy3.skel.h" 8 #include "tailcall_freplace.skel.h" 9 #include "tc_bpf2bpf.skel.h" 10 #include "tailcall_fail.skel.h" 11 #include "tailcall_cgrp_storage_owner.skel.h" 12 #include "tailcall_cgrp_storage_no_storage.skel.h" 13 #include "tailcall_cgrp_storage.skel.h" 14 #include "tailcall_sleepable.skel.h" 15 #include "tailcall_callback.skel.h" 16 #include "tailcall_bpf2bpf2.skel.h" 17 #include "tailcall_bpf2bpf_fexit.skel.h" 18 19 /* test_tailcall_1 checks basic functionality by patching multiple locations 20 * in a single program for a single tail call slot with nop->jmp, jmp->nop 21 * and jmp->jmp rewrites. Also checks for nop->nop. 22 */ 23 static void test_tailcall_1(void) 24 { 25 int err, map_fd, prog_fd, main_fd, i, j; 26 struct bpf_map *prog_array; 27 struct bpf_program *prog; 28 struct bpf_object *obj; 29 char prog_name[32]; 30 char buff[128] = {}; 31 LIBBPF_OPTS(bpf_test_run_opts, topts, 32 .data_in = buff, 33 .data_size_in = sizeof(buff), 34 .repeat = 1, 35 ); 36 37 err = bpf_prog_test_load("tailcall1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, 38 &prog_fd); 39 if (CHECK_FAIL(err)) 40 return; 41 42 prog = bpf_object__find_program_by_name(obj, "entry"); 43 if (CHECK_FAIL(!prog)) 44 goto out; 45 46 main_fd = bpf_program__fd(prog); 47 if (CHECK_FAIL(main_fd < 0)) 48 goto out; 49 50 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 51 if (CHECK_FAIL(!prog_array)) 52 goto out; 53 54 map_fd = bpf_map__fd(prog_array); 55 if (CHECK_FAIL(map_fd < 0)) 56 goto out; 57 58 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 59 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 60 61 prog = bpf_object__find_program_by_name(obj, prog_name); 62 if (CHECK_FAIL(!prog)) 63 goto out; 64 65 prog_fd = bpf_program__fd(prog); 66 if (CHECK_FAIL(prog_fd < 0)) 67 goto out; 68 69 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 70 if (CHECK_FAIL(err)) 71 goto out; 72 } 73 74 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 75 err = bpf_prog_test_run_opts(main_fd, &topts); 76 ASSERT_OK(err, "tailcall"); 77 ASSERT_EQ(topts.retval, i, "tailcall retval"); 78 79 err = bpf_map_delete_elem(map_fd, &i); 80 if (CHECK_FAIL(err)) 81 goto out; 82 } 83 84 err = bpf_prog_test_run_opts(main_fd, &topts); 85 ASSERT_OK(err, "tailcall"); 86 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 87 88 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 89 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 90 91 prog = bpf_object__find_program_by_name(obj, prog_name); 92 if (CHECK_FAIL(!prog)) 93 goto out; 94 95 prog_fd = bpf_program__fd(prog); 96 if (CHECK_FAIL(prog_fd < 0)) 97 goto out; 98 99 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 100 if (CHECK_FAIL(err)) 101 goto out; 102 } 103 104 err = bpf_prog_test_run_opts(main_fd, &topts); 105 ASSERT_OK(err, "tailcall"); 106 ASSERT_OK(topts.retval, "tailcall retval"); 107 108 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 109 j = bpf_map__max_entries(prog_array) - 1 - i; 110 snprintf(prog_name, sizeof(prog_name), "classifier_%d", j); 111 112 prog = bpf_object__find_program_by_name(obj, prog_name); 113 if (CHECK_FAIL(!prog)) 114 goto out; 115 116 prog_fd = bpf_program__fd(prog); 117 if (CHECK_FAIL(prog_fd < 0)) 118 goto out; 119 120 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 121 if (CHECK_FAIL(err)) 122 goto out; 123 } 124 125 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 126 j = bpf_map__max_entries(prog_array) - 1 - i; 127 128 err = bpf_prog_test_run_opts(main_fd, &topts); 129 ASSERT_OK(err, "tailcall"); 130 ASSERT_EQ(topts.retval, j, "tailcall retval"); 131 132 err = bpf_map_delete_elem(map_fd, &i); 133 if (CHECK_FAIL(err)) 134 goto out; 135 } 136 137 err = bpf_prog_test_run_opts(main_fd, &topts); 138 ASSERT_OK(err, "tailcall"); 139 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 140 141 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 142 err = bpf_map_delete_elem(map_fd, &i); 143 if (CHECK_FAIL(err >= 0 || errno != ENOENT)) 144 goto out; 145 146 err = bpf_prog_test_run_opts(main_fd, &topts); 147 ASSERT_OK(err, "tailcall"); 148 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 149 } 150 151 out: 152 bpf_object__close(obj); 153 } 154 155 /* test_tailcall_2 checks that patching multiple programs for a single 156 * tail call slot works. It also jumps through several programs and tests 157 * the tail call limit counter. 158 */ 159 static void test_tailcall_2(void) 160 { 161 int err, map_fd, prog_fd, main_fd, i; 162 struct bpf_map *prog_array; 163 struct bpf_program *prog; 164 struct bpf_object *obj; 165 char prog_name[32]; 166 char buff[128] = {}; 167 LIBBPF_OPTS(bpf_test_run_opts, topts, 168 .data_in = buff, 169 .data_size_in = sizeof(buff), 170 .repeat = 1, 171 ); 172 173 err = bpf_prog_test_load("tailcall2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, 174 &prog_fd); 175 if (CHECK_FAIL(err)) 176 return; 177 178 prog = bpf_object__find_program_by_name(obj, "entry"); 179 if (CHECK_FAIL(!prog)) 180 goto out; 181 182 main_fd = bpf_program__fd(prog); 183 if (CHECK_FAIL(main_fd < 0)) 184 goto out; 185 186 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 187 if (CHECK_FAIL(!prog_array)) 188 goto out; 189 190 map_fd = bpf_map__fd(prog_array); 191 if (CHECK_FAIL(map_fd < 0)) 192 goto out; 193 194 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 195 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 196 197 prog = bpf_object__find_program_by_name(obj, prog_name); 198 if (CHECK_FAIL(!prog)) 199 goto out; 200 201 prog_fd = bpf_program__fd(prog); 202 if (CHECK_FAIL(prog_fd < 0)) 203 goto out; 204 205 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 206 if (CHECK_FAIL(err)) 207 goto out; 208 } 209 210 err = bpf_prog_test_run_opts(main_fd, &topts); 211 ASSERT_OK(err, "tailcall"); 212 ASSERT_EQ(topts.retval, 2, "tailcall retval"); 213 214 i = 2; 215 err = bpf_map_delete_elem(map_fd, &i); 216 if (CHECK_FAIL(err)) 217 goto out; 218 219 err = bpf_prog_test_run_opts(main_fd, &topts); 220 ASSERT_OK(err, "tailcall"); 221 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 222 223 i = 0; 224 err = bpf_map_delete_elem(map_fd, &i); 225 if (CHECK_FAIL(err)) 226 goto out; 227 228 err = bpf_prog_test_run_opts(main_fd, &topts); 229 ASSERT_OK(err, "tailcall"); 230 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 231 out: 232 bpf_object__close(obj); 233 } 234 235 static void test_tailcall_count(const char *which, bool test_fentry, 236 bool test_fexit) 237 { 238 struct bpf_object *obj = NULL, *fentry_obj = NULL, *fexit_obj = NULL; 239 struct bpf_link *fentry_link = NULL, *fexit_link = NULL; 240 int err, map_fd, prog_fd, main_fd, data_fd, i, val; 241 struct bpf_map *prog_array, *data_map; 242 struct bpf_program *prog; 243 char buff[128] = {}; 244 LIBBPF_OPTS(bpf_test_run_opts, topts, 245 .data_in = buff, 246 .data_size_in = sizeof(buff), 247 .repeat = 1, 248 ); 249 250 err = bpf_prog_test_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj, 251 &prog_fd); 252 if (CHECK_FAIL(err)) 253 return; 254 255 prog = bpf_object__find_program_by_name(obj, "entry"); 256 if (CHECK_FAIL(!prog)) 257 goto out; 258 259 main_fd = bpf_program__fd(prog); 260 if (CHECK_FAIL(main_fd < 0)) 261 goto out; 262 263 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 264 if (CHECK_FAIL(!prog_array)) 265 goto out; 266 267 map_fd = bpf_map__fd(prog_array); 268 if (CHECK_FAIL(map_fd < 0)) 269 goto out; 270 271 prog = bpf_object__find_program_by_name(obj, "classifier_0"); 272 if (CHECK_FAIL(!prog)) 273 goto out; 274 275 prog_fd = bpf_program__fd(prog); 276 if (CHECK_FAIL(prog_fd < 0)) 277 goto out; 278 279 i = 0; 280 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 281 if (CHECK_FAIL(err)) 282 goto out; 283 284 if (test_fentry) { 285 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o", 286 NULL); 287 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file")) 288 goto out; 289 290 prog = bpf_object__find_program_by_name(fentry_obj, "fentry"); 291 if (!ASSERT_OK_PTR(prog, "find fentry prog")) 292 goto out; 293 294 err = bpf_program__set_attach_target(prog, prog_fd, 295 "subprog_tail"); 296 if (!ASSERT_OK(err, "set_attach_target subprog_tail")) 297 goto out; 298 299 err = bpf_object__load(fentry_obj); 300 if (!ASSERT_OK(err, "load fentry_obj")) 301 goto out; 302 303 fentry_link = bpf_program__attach_trace(prog); 304 if (!ASSERT_OK_PTR(fentry_link, "attach_trace")) 305 goto out; 306 } 307 308 if (test_fexit) { 309 fexit_obj = bpf_object__open_file("tailcall_bpf2bpf_fexit.bpf.o", 310 NULL); 311 if (!ASSERT_OK_PTR(fexit_obj, "open fexit_obj file")) 312 goto out; 313 314 prog = bpf_object__find_program_by_name(fexit_obj, "fexit"); 315 if (!ASSERT_OK_PTR(prog, "find fexit prog")) 316 goto out; 317 318 err = bpf_program__set_attach_target(prog, prog_fd, 319 "subprog_tail"); 320 if (!ASSERT_OK(err, "set_attach_target subprog_tail")) 321 goto out; 322 323 err = bpf_object__load(fexit_obj); 324 if (!ASSERT_OK(err, "load fexit_obj")) 325 goto out; 326 327 fexit_link = bpf_program__attach_trace(prog); 328 if (!ASSERT_OK_PTR(fexit_link, "attach_trace")) 329 goto out; 330 } 331 332 err = bpf_prog_test_run_opts(main_fd, &topts); 333 ASSERT_OK(err, "tailcall"); 334 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 335 336 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 337 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 338 goto out; 339 340 data_fd = bpf_map__fd(data_map); 341 if (CHECK_FAIL(data_fd < 0)) 342 goto out; 343 344 i = 0; 345 err = bpf_map_lookup_elem(data_fd, &i, &val); 346 ASSERT_OK(err, "tailcall count"); 347 ASSERT_EQ(val, 33, "tailcall count"); 348 349 if (test_fentry) { 350 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss"); 351 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 352 "find tailcall_bpf2bpf_fentry.bss map")) 353 goto out; 354 355 data_fd = bpf_map__fd(data_map); 356 if (!ASSERT_FALSE(data_fd < 0, 357 "find tailcall_bpf2bpf_fentry.bss map fd")) 358 goto out; 359 360 i = 0; 361 err = bpf_map_lookup_elem(data_fd, &i, &val); 362 ASSERT_OK(err, "fentry count"); 363 ASSERT_EQ(val, 33, "fentry count"); 364 } 365 366 if (test_fexit) { 367 data_map = bpf_object__find_map_by_name(fexit_obj, ".bss"); 368 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 369 "find tailcall_bpf2bpf_fexit.bss map")) 370 goto out; 371 372 data_fd = bpf_map__fd(data_map); 373 if (!ASSERT_FALSE(data_fd < 0, 374 "find tailcall_bpf2bpf_fexit.bss map fd")) 375 goto out; 376 377 i = 0; 378 err = bpf_map_lookup_elem(data_fd, &i, &val); 379 ASSERT_OK(err, "fexit count"); 380 ASSERT_EQ(val, 33, "fexit count"); 381 } 382 383 i = 0; 384 err = bpf_map_delete_elem(map_fd, &i); 385 if (CHECK_FAIL(err)) 386 goto out; 387 388 err = bpf_prog_test_run_opts(main_fd, &topts); 389 ASSERT_OK(err, "tailcall"); 390 ASSERT_OK(topts.retval, "tailcall retval"); 391 out: 392 bpf_link__destroy(fentry_link); 393 bpf_link__destroy(fexit_link); 394 bpf_object__close(fentry_obj); 395 bpf_object__close(fexit_obj); 396 bpf_object__close(obj); 397 } 398 399 /* test_tailcall_3 checks that the count value of the tail call limit 400 * enforcement matches with expectations. JIT uses direct jump. 401 */ 402 static void test_tailcall_3(void) 403 { 404 test_tailcall_count("tailcall3.bpf.o", false, false); 405 } 406 407 /* test_tailcall_6 checks that the count value of the tail call limit 408 * enforcement matches with expectations. JIT uses indirect jump. 409 */ 410 static void test_tailcall_6(void) 411 { 412 test_tailcall_count("tailcall6.bpf.o", false, false); 413 } 414 415 /* test_tailcall_4 checks that the kernel properly selects indirect jump 416 * for the case where the key is not known. Latter is passed via global 417 * data to select different targets we can compare return value of. 418 */ 419 static void test_tailcall_4(void) 420 { 421 int err, map_fd, prog_fd, main_fd, data_fd, i; 422 struct bpf_map *prog_array, *data_map; 423 struct bpf_program *prog; 424 struct bpf_object *obj; 425 static const int zero = 0; 426 char buff[128] = {}; 427 char prog_name[32]; 428 LIBBPF_OPTS(bpf_test_run_opts, topts, 429 .data_in = buff, 430 .data_size_in = sizeof(buff), 431 .repeat = 1, 432 ); 433 434 err = bpf_prog_test_load("tailcall4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, 435 &prog_fd); 436 if (CHECK_FAIL(err)) 437 return; 438 439 prog = bpf_object__find_program_by_name(obj, "entry"); 440 if (CHECK_FAIL(!prog)) 441 goto out; 442 443 main_fd = bpf_program__fd(prog); 444 if (CHECK_FAIL(main_fd < 0)) 445 goto out; 446 447 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 448 if (CHECK_FAIL(!prog_array)) 449 goto out; 450 451 map_fd = bpf_map__fd(prog_array); 452 if (CHECK_FAIL(map_fd < 0)) 453 goto out; 454 455 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 456 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 457 goto out; 458 459 data_fd = bpf_map__fd(data_map); 460 if (CHECK_FAIL(data_fd < 0)) 461 goto out; 462 463 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 464 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 465 466 prog = bpf_object__find_program_by_name(obj, prog_name); 467 if (CHECK_FAIL(!prog)) 468 goto out; 469 470 prog_fd = bpf_program__fd(prog); 471 if (CHECK_FAIL(prog_fd < 0)) 472 goto out; 473 474 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 475 if (CHECK_FAIL(err)) 476 goto out; 477 } 478 479 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 480 err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY); 481 if (CHECK_FAIL(err)) 482 goto out; 483 484 err = bpf_prog_test_run_opts(main_fd, &topts); 485 ASSERT_OK(err, "tailcall"); 486 ASSERT_EQ(topts.retval, i, "tailcall retval"); 487 } 488 489 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 490 err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY); 491 if (CHECK_FAIL(err)) 492 goto out; 493 494 err = bpf_map_delete_elem(map_fd, &i); 495 if (CHECK_FAIL(err)) 496 goto out; 497 498 err = bpf_prog_test_run_opts(main_fd, &topts); 499 ASSERT_OK(err, "tailcall"); 500 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 501 } 502 out: 503 bpf_object__close(obj); 504 } 505 506 /* test_tailcall_5 probes similarly to test_tailcall_4 that the kernel generates 507 * an indirect jump when the keys are const but different from different branches. 508 */ 509 static void test_tailcall_5(void) 510 { 511 int err, map_fd, prog_fd, main_fd, data_fd, i, key[] = { 1111, 1234, 5678 }; 512 struct bpf_map *prog_array, *data_map; 513 struct bpf_program *prog; 514 struct bpf_object *obj; 515 static const int zero = 0; 516 char buff[128] = {}; 517 char prog_name[32]; 518 LIBBPF_OPTS(bpf_test_run_opts, topts, 519 .data_in = buff, 520 .data_size_in = sizeof(buff), 521 .repeat = 1, 522 ); 523 524 err = bpf_prog_test_load("tailcall5.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, 525 &prog_fd); 526 if (CHECK_FAIL(err)) 527 return; 528 529 prog = bpf_object__find_program_by_name(obj, "entry"); 530 if (CHECK_FAIL(!prog)) 531 goto out; 532 533 main_fd = bpf_program__fd(prog); 534 if (CHECK_FAIL(main_fd < 0)) 535 goto out; 536 537 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 538 if (CHECK_FAIL(!prog_array)) 539 goto out; 540 541 map_fd = bpf_map__fd(prog_array); 542 if (CHECK_FAIL(map_fd < 0)) 543 goto out; 544 545 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 546 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 547 goto out; 548 549 data_fd = bpf_map__fd(data_map); 550 if (CHECK_FAIL(data_fd < 0)) 551 goto out; 552 553 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 554 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 555 556 prog = bpf_object__find_program_by_name(obj, prog_name); 557 if (CHECK_FAIL(!prog)) 558 goto out; 559 560 prog_fd = bpf_program__fd(prog); 561 if (CHECK_FAIL(prog_fd < 0)) 562 goto out; 563 564 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 565 if (CHECK_FAIL(err)) 566 goto out; 567 } 568 569 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 570 err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY); 571 if (CHECK_FAIL(err)) 572 goto out; 573 574 err = bpf_prog_test_run_opts(main_fd, &topts); 575 ASSERT_OK(err, "tailcall"); 576 ASSERT_EQ(topts.retval, i, "tailcall retval"); 577 } 578 579 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 580 err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY); 581 if (CHECK_FAIL(err)) 582 goto out; 583 584 err = bpf_map_delete_elem(map_fd, &i); 585 if (CHECK_FAIL(err)) 586 goto out; 587 588 err = bpf_prog_test_run_opts(main_fd, &topts); 589 ASSERT_OK(err, "tailcall"); 590 ASSERT_EQ(topts.retval, 3, "tailcall retval"); 591 } 592 out: 593 bpf_object__close(obj); 594 } 595 596 /* test_tailcall_bpf2bpf_1 purpose is to make sure that tailcalls are working 597 * correctly in correlation with BPF subprograms 598 */ 599 static void test_tailcall_bpf2bpf_1(void) 600 { 601 int err, map_fd, prog_fd, main_fd, i; 602 struct bpf_map *prog_array; 603 struct bpf_program *prog; 604 struct bpf_object *obj; 605 char prog_name[32]; 606 LIBBPF_OPTS(bpf_test_run_opts, topts, 607 .data_in = &pkt_v4, 608 .data_size_in = sizeof(pkt_v4), 609 .repeat = 1, 610 ); 611 612 err = bpf_prog_test_load("tailcall_bpf2bpf1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, 613 &obj, &prog_fd); 614 if (CHECK_FAIL(err)) 615 return; 616 617 prog = bpf_object__find_program_by_name(obj, "entry"); 618 if (CHECK_FAIL(!prog)) 619 goto out; 620 621 main_fd = bpf_program__fd(prog); 622 if (CHECK_FAIL(main_fd < 0)) 623 goto out; 624 625 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 626 if (CHECK_FAIL(!prog_array)) 627 goto out; 628 629 map_fd = bpf_map__fd(prog_array); 630 if (CHECK_FAIL(map_fd < 0)) 631 goto out; 632 633 /* nop -> jmp */ 634 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 635 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 636 637 prog = bpf_object__find_program_by_name(obj, prog_name); 638 if (CHECK_FAIL(!prog)) 639 goto out; 640 641 prog_fd = bpf_program__fd(prog); 642 if (CHECK_FAIL(prog_fd < 0)) 643 goto out; 644 645 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 646 if (CHECK_FAIL(err)) 647 goto out; 648 } 649 650 err = bpf_prog_test_run_opts(main_fd, &topts); 651 ASSERT_OK(err, "tailcall"); 652 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 653 654 /* jmp -> nop, call subprog that will do tailcall */ 655 i = 1; 656 err = bpf_map_delete_elem(map_fd, &i); 657 if (CHECK_FAIL(err)) 658 goto out; 659 660 err = bpf_prog_test_run_opts(main_fd, &topts); 661 ASSERT_OK(err, "tailcall"); 662 ASSERT_OK(topts.retval, "tailcall retval"); 663 664 /* make sure that subprog can access ctx and entry prog that 665 * called this subprog can properly return 666 */ 667 i = 0; 668 err = bpf_map_delete_elem(map_fd, &i); 669 if (CHECK_FAIL(err)) 670 goto out; 671 672 err = bpf_prog_test_run_opts(main_fd, &topts); 673 ASSERT_OK(err, "tailcall"); 674 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 2, "tailcall retval"); 675 out: 676 bpf_object__close(obj); 677 } 678 679 /* test_tailcall_bpf2bpf_2 checks that the count value of the tail call limit 680 * enforcement matches with expectations when tailcall is preceded with 681 * bpf2bpf call. 682 */ 683 static void test_tailcall_bpf2bpf_2(void) 684 { 685 int err, map_fd, prog_fd, main_fd, data_fd, i, val; 686 struct bpf_map *prog_array, *data_map; 687 struct bpf_program *prog; 688 struct bpf_object *obj; 689 char buff[128] = {}; 690 LIBBPF_OPTS(bpf_test_run_opts, topts, 691 .data_in = buff, 692 .data_size_in = sizeof(buff), 693 .repeat = 1, 694 ); 695 696 err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, 697 &obj, &prog_fd); 698 if (CHECK_FAIL(err)) 699 return; 700 701 prog = bpf_object__find_program_by_name(obj, "entry"); 702 if (CHECK_FAIL(!prog)) 703 goto out; 704 705 main_fd = bpf_program__fd(prog); 706 if (CHECK_FAIL(main_fd < 0)) 707 goto out; 708 709 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 710 if (CHECK_FAIL(!prog_array)) 711 goto out; 712 713 map_fd = bpf_map__fd(prog_array); 714 if (CHECK_FAIL(map_fd < 0)) 715 goto out; 716 717 prog = bpf_object__find_program_by_name(obj, "classifier_0"); 718 if (CHECK_FAIL(!prog)) 719 goto out; 720 721 prog_fd = bpf_program__fd(prog); 722 if (CHECK_FAIL(prog_fd < 0)) 723 goto out; 724 725 i = 0; 726 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 727 if (CHECK_FAIL(err)) 728 goto out; 729 730 err = bpf_prog_test_run_opts(main_fd, &topts); 731 ASSERT_OK(err, "tailcall"); 732 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 733 734 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 735 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 736 goto out; 737 738 data_fd = bpf_map__fd(data_map); 739 if (CHECK_FAIL(data_fd < 0)) 740 goto out; 741 742 i = 0; 743 err = bpf_map_lookup_elem(data_fd, &i, &val); 744 ASSERT_OK(err, "tailcall count"); 745 ASSERT_EQ(val, 33, "tailcall count"); 746 747 i = 0; 748 err = bpf_map_delete_elem(map_fd, &i); 749 if (CHECK_FAIL(err)) 750 goto out; 751 752 err = bpf_prog_test_run_opts(main_fd, &topts); 753 ASSERT_OK(err, "tailcall"); 754 ASSERT_OK(topts.retval, "tailcall retval"); 755 out: 756 bpf_object__close(obj); 757 } 758 759 /* test_tailcall_bpf2bpf_3 checks that non-trivial amount of stack (up to 760 * 256 bytes) can be used within bpf subprograms that have the tailcalls 761 * in them 762 */ 763 static void test_tailcall_bpf2bpf_3(void) 764 { 765 int err, map_fd, prog_fd, main_fd, i; 766 struct bpf_map *prog_array; 767 struct bpf_program *prog; 768 struct bpf_object *obj; 769 char prog_name[32]; 770 LIBBPF_OPTS(bpf_test_run_opts, topts, 771 .data_in = &pkt_v4, 772 .data_size_in = sizeof(pkt_v4), 773 .repeat = 1, 774 ); 775 776 err = bpf_prog_test_load("tailcall_bpf2bpf3.bpf.o", BPF_PROG_TYPE_SCHED_CLS, 777 &obj, &prog_fd); 778 if (CHECK_FAIL(err)) 779 return; 780 781 prog = bpf_object__find_program_by_name(obj, "entry"); 782 if (CHECK_FAIL(!prog)) 783 goto out; 784 785 main_fd = bpf_program__fd(prog); 786 if (CHECK_FAIL(main_fd < 0)) 787 goto out; 788 789 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 790 if (CHECK_FAIL(!prog_array)) 791 goto out; 792 793 map_fd = bpf_map__fd(prog_array); 794 if (CHECK_FAIL(map_fd < 0)) 795 goto out; 796 797 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 798 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 799 800 prog = bpf_object__find_program_by_name(obj, prog_name); 801 if (CHECK_FAIL(!prog)) 802 goto out; 803 804 prog_fd = bpf_program__fd(prog); 805 if (CHECK_FAIL(prog_fd < 0)) 806 goto out; 807 808 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 809 if (CHECK_FAIL(err)) 810 goto out; 811 } 812 813 err = bpf_prog_test_run_opts(main_fd, &topts); 814 ASSERT_OK(err, "tailcall"); 815 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 3, "tailcall retval"); 816 817 i = 1; 818 err = bpf_map_delete_elem(map_fd, &i); 819 if (CHECK_FAIL(err)) 820 goto out; 821 822 err = bpf_prog_test_run_opts(main_fd, &topts); 823 ASSERT_OK(err, "tailcall"); 824 ASSERT_EQ(topts.retval, sizeof(pkt_v4), "tailcall retval"); 825 826 i = 0; 827 err = bpf_map_delete_elem(map_fd, &i); 828 if (CHECK_FAIL(err)) 829 goto out; 830 831 err = bpf_prog_test_run_opts(main_fd, &topts); 832 ASSERT_OK(err, "tailcall"); 833 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 2, "tailcall retval"); 834 out: 835 bpf_object__close(obj); 836 } 837 838 #include "tailcall_bpf2bpf4.skel.h" 839 840 /* test_tailcall_bpf2bpf_4 checks that tailcall counter is correctly preserved 841 * across tailcalls combined with bpf2bpf calls. for making sure that tailcall 842 * counter behaves correctly, bpf program will go through following flow: 843 * 844 * entry -> entry_subprog -> tailcall0 -> bpf_func0 -> subprog0 -> 845 * -> tailcall1 -> bpf_func1 -> subprog1 -> tailcall2 -> bpf_func2 -> 846 * subprog2 [here bump global counter] --------^ 847 * 848 * We go through first two tailcalls and start counting from the subprog2 where 849 * the loop begins. At the end of the test make sure that the global counter is 850 * equal to 31, because tailcall counter includes the first two tailcalls 851 * whereas global counter is incremented only on loop presented on flow above. 852 * 853 * The noise parameter is used to insert bpf_map_update calls into the logic 854 * to force verifier to patch instructions. This allows us to ensure jump 855 * logic remains correct with instruction movement. 856 */ 857 static void test_tailcall_bpf2bpf_4(bool noise) 858 { 859 int err, map_fd, prog_fd, main_fd, data_fd, i; 860 struct tailcall_bpf2bpf4__bss val; 861 struct bpf_map *prog_array, *data_map; 862 struct bpf_program *prog; 863 struct bpf_object *obj; 864 char prog_name[32]; 865 LIBBPF_OPTS(bpf_test_run_opts, topts, 866 .data_in = &pkt_v4, 867 .data_size_in = sizeof(pkt_v4), 868 .repeat = 1, 869 ); 870 871 err = bpf_prog_test_load("tailcall_bpf2bpf4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, 872 &obj, &prog_fd); 873 if (CHECK_FAIL(err)) 874 return; 875 876 prog = bpf_object__find_program_by_name(obj, "entry"); 877 if (CHECK_FAIL(!prog)) 878 goto out; 879 880 main_fd = bpf_program__fd(prog); 881 if (CHECK_FAIL(main_fd < 0)) 882 goto out; 883 884 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 885 if (CHECK_FAIL(!prog_array)) 886 goto out; 887 888 map_fd = bpf_map__fd(prog_array); 889 if (CHECK_FAIL(map_fd < 0)) 890 goto out; 891 892 for (i = 0; i < bpf_map__max_entries(prog_array); i++) { 893 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i); 894 895 prog = bpf_object__find_program_by_name(obj, prog_name); 896 if (CHECK_FAIL(!prog)) 897 goto out; 898 899 prog_fd = bpf_program__fd(prog); 900 if (CHECK_FAIL(prog_fd < 0)) 901 goto out; 902 903 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 904 if (CHECK_FAIL(err)) 905 goto out; 906 } 907 908 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss"); 909 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map))) 910 goto out; 911 912 data_fd = bpf_map__fd(data_map); 913 if (CHECK_FAIL(data_fd < 0)) 914 goto out; 915 916 i = 0; 917 val.noise = noise; 918 val.count = 0; 919 err = bpf_map_update_elem(data_fd, &i, &val, BPF_ANY); 920 if (CHECK_FAIL(err)) 921 goto out; 922 923 err = bpf_prog_test_run_opts(main_fd, &topts); 924 ASSERT_OK(err, "tailcall"); 925 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 3, "tailcall retval"); 926 927 i = 0; 928 err = bpf_map_lookup_elem(data_fd, &i, &val); 929 ASSERT_OK(err, "tailcall count"); 930 ASSERT_EQ(val.count, 31, "tailcall count"); 931 932 out: 933 bpf_object__close(obj); 934 } 935 936 #include "tailcall_bpf2bpf6.skel.h" 937 938 /* Tail call counting works even when there is data on stack which is 939 * not aligned to 8 bytes. 940 */ 941 static void test_tailcall_bpf2bpf_6(void) 942 { 943 struct tailcall_bpf2bpf6 *obj; 944 int err, map_fd, prog_fd, main_fd, data_fd, i, val; 945 LIBBPF_OPTS(bpf_test_run_opts, topts, 946 .data_in = &pkt_v4, 947 .data_size_in = sizeof(pkt_v4), 948 .repeat = 1, 949 ); 950 951 obj = tailcall_bpf2bpf6__open_and_load(); 952 if (!ASSERT_OK_PTR(obj, "open and load")) 953 return; 954 955 main_fd = bpf_program__fd(obj->progs.entry); 956 if (!ASSERT_GE(main_fd, 0, "entry prog fd")) 957 goto out; 958 959 map_fd = bpf_map__fd(obj->maps.jmp_table); 960 if (!ASSERT_GE(map_fd, 0, "jmp_table map fd")) 961 goto out; 962 963 prog_fd = bpf_program__fd(obj->progs.classifier_0); 964 if (!ASSERT_GE(prog_fd, 0, "classifier_0 prog fd")) 965 goto out; 966 967 i = 0; 968 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 969 if (!ASSERT_OK(err, "jmp_table map update")) 970 goto out; 971 972 err = bpf_prog_test_run_opts(main_fd, &topts); 973 ASSERT_OK(err, "entry prog test run"); 974 ASSERT_EQ(topts.retval, 0, "tailcall retval"); 975 976 data_fd = bpf_map__fd(obj->maps.bss); 977 if (!ASSERT_GE(data_fd, 0, "bss map fd")) 978 goto out; 979 980 i = 0; 981 err = bpf_map_lookup_elem(data_fd, &i, &val); 982 ASSERT_OK(err, "bss map lookup"); 983 ASSERT_EQ(val, 1, "done flag is set"); 984 985 out: 986 tailcall_bpf2bpf6__destroy(obj); 987 } 988 989 /* test_tailcall_bpf2bpf_fentry checks that the count value of the tail call 990 * limit enforcement matches with expectations when tailcall is preceded with 991 * bpf2bpf call, and the bpf2bpf call is traced by fentry. 992 */ 993 static void test_tailcall_bpf2bpf_fentry(void) 994 { 995 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", true, false); 996 } 997 998 /* test_tailcall_bpf2bpf_fexit checks that the count value of the tail call 999 * limit enforcement matches with expectations when tailcall is preceded with 1000 * bpf2bpf call, and the bpf2bpf call is traced by fexit. 1001 */ 1002 static void test_tailcall_bpf2bpf_fexit(void) 1003 { 1004 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", false, true); 1005 } 1006 1007 /* test_tailcall_bpf2bpf_fentry_fexit checks that the count value of the tail 1008 * call limit enforcement matches with expectations when tailcall is preceded 1009 * with bpf2bpf call, and the bpf2bpf call is traced by both fentry and fexit. 1010 */ 1011 static void test_tailcall_bpf2bpf_fentry_fexit(void) 1012 { 1013 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", true, true); 1014 } 1015 1016 /* test_tailcall_bpf2bpf_fentry_entry checks that the count value of the tail 1017 * call limit enforcement matches with expectations when tailcall is preceded 1018 * with bpf2bpf call, and the bpf2bpf caller is traced by fentry. 1019 */ 1020 static void test_tailcall_bpf2bpf_fentry_entry(void) 1021 { 1022 struct bpf_object *tgt_obj = NULL, *fentry_obj = NULL; 1023 int err, map_fd, prog_fd, data_fd, i, val; 1024 struct bpf_map *prog_array, *data_map; 1025 struct bpf_link *fentry_link = NULL; 1026 struct bpf_program *prog; 1027 char buff[128] = {}; 1028 1029 LIBBPF_OPTS(bpf_test_run_opts, topts, 1030 .data_in = buff, 1031 .data_size_in = sizeof(buff), 1032 .repeat = 1, 1033 ); 1034 1035 err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o", 1036 BPF_PROG_TYPE_SCHED_CLS, 1037 &tgt_obj, &prog_fd); 1038 if (!ASSERT_OK(err, "load tgt_obj")) 1039 return; 1040 1041 prog_array = bpf_object__find_map_by_name(tgt_obj, "jmp_table"); 1042 if (!ASSERT_OK_PTR(prog_array, "find jmp_table map")) 1043 goto out; 1044 1045 map_fd = bpf_map__fd(prog_array); 1046 if (!ASSERT_FALSE(map_fd < 0, "find jmp_table map fd")) 1047 goto out; 1048 1049 prog = bpf_object__find_program_by_name(tgt_obj, "classifier_0"); 1050 if (!ASSERT_OK_PTR(prog, "find classifier_0 prog")) 1051 goto out; 1052 1053 prog_fd = bpf_program__fd(prog); 1054 if (!ASSERT_FALSE(prog_fd < 0, "find classifier_0 prog fd")) 1055 goto out; 1056 1057 i = 0; 1058 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 1059 if (!ASSERT_OK(err, "update jmp_table")) 1060 goto out; 1061 1062 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o", 1063 NULL); 1064 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file")) 1065 goto out; 1066 1067 prog = bpf_object__find_program_by_name(fentry_obj, "fentry"); 1068 if (!ASSERT_OK_PTR(prog, "find fentry prog")) 1069 goto out; 1070 1071 err = bpf_program__set_attach_target(prog, prog_fd, "classifier_0"); 1072 if (!ASSERT_OK(err, "set_attach_target classifier_0")) 1073 goto out; 1074 1075 err = bpf_object__load(fentry_obj); 1076 if (!ASSERT_OK(err, "load fentry_obj")) 1077 goto out; 1078 1079 fentry_link = bpf_program__attach_trace(prog); 1080 if (!ASSERT_OK_PTR(fentry_link, "attach_trace")) 1081 goto out; 1082 1083 err = bpf_prog_test_run_opts(prog_fd, &topts); 1084 ASSERT_OK(err, "tailcall"); 1085 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 1086 1087 data_map = bpf_object__find_map_by_name(tgt_obj, "tailcall.bss"); 1088 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 1089 "find tailcall.bss map")) 1090 goto out; 1091 1092 data_fd = bpf_map__fd(data_map); 1093 if (!ASSERT_FALSE(data_fd < 0, "find tailcall.bss map fd")) 1094 goto out; 1095 1096 i = 0; 1097 err = bpf_map_lookup_elem(data_fd, &i, &val); 1098 ASSERT_OK(err, "tailcall count"); 1099 ASSERT_EQ(val, 34, "tailcall count"); 1100 1101 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss"); 1102 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 1103 "find tailcall_bpf2bpf_fentry.bss map")) 1104 goto out; 1105 1106 data_fd = bpf_map__fd(data_map); 1107 if (!ASSERT_FALSE(data_fd < 0, 1108 "find tailcall_bpf2bpf_fentry.bss map fd")) 1109 goto out; 1110 1111 i = 0; 1112 err = bpf_map_lookup_elem(data_fd, &i, &val); 1113 ASSERT_OK(err, "fentry count"); 1114 ASSERT_EQ(val, 1, "fentry count"); 1115 1116 out: 1117 bpf_link__destroy(fentry_link); 1118 bpf_object__close(fentry_obj); 1119 bpf_object__close(tgt_obj); 1120 } 1121 1122 #define JMP_TABLE "/sys/fs/bpf/jmp_table" 1123 1124 static int poke_thread_exit; 1125 1126 static void *poke_update(void *arg) 1127 { 1128 __u32 zero = 0, prog1_fd, prog2_fd, map_fd; 1129 struct tailcall_poke *call = arg; 1130 1131 map_fd = bpf_map__fd(call->maps.jmp_table); 1132 prog1_fd = bpf_program__fd(call->progs.call1); 1133 prog2_fd = bpf_program__fd(call->progs.call2); 1134 1135 while (!poke_thread_exit) { 1136 bpf_map_update_elem(map_fd, &zero, &prog1_fd, BPF_ANY); 1137 bpf_map_update_elem(map_fd, &zero, &prog2_fd, BPF_ANY); 1138 } 1139 1140 return NULL; 1141 } 1142 1143 /* 1144 * We are trying to hit prog array update during another program load 1145 * that shares the same prog array map. 1146 * 1147 * For that we share the jmp_table map between two skeleton instances 1148 * by pinning the jmp_table to same path. Then first skeleton instance 1149 * periodically updates jmp_table in 'poke update' thread while we load 1150 * the second skeleton instance in the main thread. 1151 */ 1152 static void test_tailcall_poke(void) 1153 { 1154 struct tailcall_poke *call, *test; 1155 int err, cnt = 10; 1156 pthread_t thread; 1157 1158 unlink(JMP_TABLE); 1159 1160 call = tailcall_poke__open_and_load(); 1161 if (!ASSERT_OK_PTR(call, "tailcall_poke__open")) 1162 return; 1163 1164 err = bpf_map__pin(call->maps.jmp_table, JMP_TABLE); 1165 if (!ASSERT_OK(err, "bpf_map__pin")) 1166 goto out; 1167 1168 err = pthread_create(&thread, NULL, poke_update, call); 1169 if (!ASSERT_OK(err, "new toggler")) 1170 goto out; 1171 1172 while (cnt--) { 1173 test = tailcall_poke__open(); 1174 if (!ASSERT_OK_PTR(test, "tailcall_poke__open")) 1175 break; 1176 1177 err = bpf_map__set_pin_path(test->maps.jmp_table, JMP_TABLE); 1178 if (!ASSERT_OK(err, "bpf_map__pin")) { 1179 tailcall_poke__destroy(test); 1180 break; 1181 } 1182 1183 bpf_program__set_autoload(test->progs.test, true); 1184 bpf_program__set_autoload(test->progs.call1, false); 1185 bpf_program__set_autoload(test->progs.call2, false); 1186 1187 err = tailcall_poke__load(test); 1188 tailcall_poke__destroy(test); 1189 if (!ASSERT_OK(err, "tailcall_poke__load")) 1190 break; 1191 } 1192 1193 poke_thread_exit = 1; 1194 ASSERT_OK(pthread_join(thread, NULL), "pthread_join"); 1195 1196 out: 1197 bpf_map__unpin(call->maps.jmp_table, JMP_TABLE); 1198 tailcall_poke__destroy(call); 1199 } 1200 1201 static void test_tailcall_hierarchy_count(const char *which, bool test_fentry, 1202 bool test_fexit, 1203 bool test_fentry_entry) 1204 { 1205 int err, map_fd, prog_fd, main_data_fd, fentry_data_fd = 0, fexit_data_fd = 0, i, val; 1206 struct bpf_object *obj = NULL, *fentry_obj = NULL, *fexit_obj = NULL; 1207 struct bpf_link *fentry_link = NULL, *fexit_link = NULL; 1208 struct bpf_program *prog, *fentry_prog; 1209 struct bpf_map *prog_array, *data_map; 1210 int fentry_prog_fd; 1211 char buff[128] = {}; 1212 1213 LIBBPF_OPTS(bpf_test_run_opts, topts, 1214 .data_in = buff, 1215 .data_size_in = sizeof(buff), 1216 .repeat = 1, 1217 ); 1218 1219 err = bpf_prog_test_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj, 1220 &prog_fd); 1221 if (!ASSERT_OK(err, "load obj")) 1222 return; 1223 1224 prog = bpf_object__find_program_by_name(obj, "entry"); 1225 if (!ASSERT_OK_PTR(prog, "find entry prog")) 1226 goto out; 1227 1228 prog_fd = bpf_program__fd(prog); 1229 if (!ASSERT_GE(prog_fd, 0, "prog_fd")) 1230 goto out; 1231 1232 if (test_fentry_entry) { 1233 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_hierarchy_fentry.bpf.o", 1234 NULL); 1235 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file")) 1236 goto out; 1237 1238 fentry_prog = bpf_object__find_program_by_name(fentry_obj, 1239 "fentry"); 1240 if (!ASSERT_OK_PTR(prog, "find fentry prog")) 1241 goto out; 1242 1243 err = bpf_program__set_attach_target(fentry_prog, prog_fd, 1244 "entry"); 1245 if (!ASSERT_OK(err, "set_attach_target entry")) 1246 goto out; 1247 1248 err = bpf_object__load(fentry_obj); 1249 if (!ASSERT_OK(err, "load fentry_obj")) 1250 goto out; 1251 1252 fentry_link = bpf_program__attach_trace(fentry_prog); 1253 if (!ASSERT_OK_PTR(fentry_link, "attach_trace")) 1254 goto out; 1255 1256 fentry_prog_fd = bpf_program__fd(fentry_prog); 1257 if (!ASSERT_GE(fentry_prog_fd, 0, "fentry_prog_fd")) 1258 goto out; 1259 1260 prog_array = bpf_object__find_map_by_name(fentry_obj, "jmp_table"); 1261 if (!ASSERT_OK_PTR(prog_array, "find jmp_table")) 1262 goto out; 1263 1264 map_fd = bpf_map__fd(prog_array); 1265 if (!ASSERT_GE(map_fd, 0, "map_fd")) 1266 goto out; 1267 1268 i = 0; 1269 err = bpf_map_update_elem(map_fd, &i, &fentry_prog_fd, BPF_ANY); 1270 if (!ASSERT_OK(err, "update jmp_table")) 1271 goto out; 1272 1273 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss"); 1274 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 1275 "find data_map")) 1276 goto out; 1277 1278 } else { 1279 prog_array = bpf_object__find_map_by_name(obj, "jmp_table"); 1280 if (!ASSERT_OK_PTR(prog_array, "find jmp_table")) 1281 goto out; 1282 1283 map_fd = bpf_map__fd(prog_array); 1284 if (!ASSERT_GE(map_fd, 0, "map_fd")) 1285 goto out; 1286 1287 i = 0; 1288 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY); 1289 if (!ASSERT_OK(err, "update jmp_table")) 1290 goto out; 1291 1292 data_map = bpf_object__find_map_by_name(obj, ".bss"); 1293 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 1294 "find data_map")) 1295 goto out; 1296 } 1297 1298 if (test_fentry) { 1299 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o", 1300 NULL); 1301 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file")) 1302 goto out; 1303 1304 prog = bpf_object__find_program_by_name(fentry_obj, "fentry"); 1305 if (!ASSERT_OK_PTR(prog, "find fentry prog")) 1306 goto out; 1307 1308 err = bpf_program__set_attach_target(prog, prog_fd, 1309 "subprog_tail"); 1310 if (!ASSERT_OK(err, "set_attach_target subprog_tail")) 1311 goto out; 1312 1313 err = bpf_object__load(fentry_obj); 1314 if (!ASSERT_OK(err, "load fentry_obj")) 1315 goto out; 1316 1317 fentry_link = bpf_program__attach_trace(prog); 1318 if (!ASSERT_OK_PTR(fentry_link, "attach_trace")) 1319 goto out; 1320 } 1321 1322 if (test_fexit) { 1323 fexit_obj = bpf_object__open_file("tailcall_bpf2bpf_fexit.bpf.o", 1324 NULL); 1325 if (!ASSERT_OK_PTR(fexit_obj, "open fexit_obj file")) 1326 goto out; 1327 1328 prog = bpf_object__find_program_by_name(fexit_obj, "fexit"); 1329 if (!ASSERT_OK_PTR(prog, "find fexit prog")) 1330 goto out; 1331 1332 err = bpf_program__set_attach_target(prog, prog_fd, 1333 "subprog_tail"); 1334 if (!ASSERT_OK(err, "set_attach_target subprog_tail")) 1335 goto out; 1336 1337 err = bpf_object__load(fexit_obj); 1338 if (!ASSERT_OK(err, "load fexit_obj")) 1339 goto out; 1340 1341 fexit_link = bpf_program__attach_trace(prog); 1342 if (!ASSERT_OK_PTR(fexit_link, "attach_trace")) 1343 goto out; 1344 } 1345 1346 err = bpf_prog_test_run_opts(prog_fd, &topts); 1347 ASSERT_OK(err, "tailcall"); 1348 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 1349 1350 main_data_fd = bpf_map__fd(data_map); 1351 if (!ASSERT_GE(main_data_fd, 0, "main_data_fd")) 1352 goto out; 1353 1354 i = 0; 1355 err = bpf_map_lookup_elem(main_data_fd, &i, &val); 1356 ASSERT_OK(err, "tailcall count"); 1357 ASSERT_EQ(val, 34, "tailcall count"); 1358 1359 if (test_fentry) { 1360 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss"); 1361 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 1362 "find tailcall_bpf2bpf_fentry.bss map")) 1363 goto out; 1364 1365 fentry_data_fd = bpf_map__fd(data_map); 1366 if (!ASSERT_GE(fentry_data_fd, 0, 1367 "find tailcall_bpf2bpf_fentry.bss map fd")) 1368 goto out; 1369 1370 i = 0; 1371 err = bpf_map_lookup_elem(fentry_data_fd, &i, &val); 1372 ASSERT_OK(err, "fentry count"); 1373 ASSERT_EQ(val, 68, "fentry count"); 1374 } 1375 1376 if (test_fexit) { 1377 data_map = bpf_object__find_map_by_name(fexit_obj, ".bss"); 1378 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map), 1379 "find tailcall_bpf2bpf_fexit.bss map")) 1380 goto out; 1381 1382 fexit_data_fd = bpf_map__fd(data_map); 1383 if (!ASSERT_GE(fexit_data_fd, 0, 1384 "find tailcall_bpf2bpf_fexit.bss map fd")) 1385 goto out; 1386 1387 i = 0; 1388 err = bpf_map_lookup_elem(fexit_data_fd, &i, &val); 1389 ASSERT_OK(err, "fexit count"); 1390 ASSERT_EQ(val, 68, "fexit count"); 1391 } 1392 1393 i = 0; 1394 err = bpf_map_delete_elem(map_fd, &i); 1395 if (!ASSERT_OK(err, "delete_elem from jmp_table")) 1396 goto out; 1397 1398 err = bpf_prog_test_run_opts(prog_fd, &topts); 1399 ASSERT_OK(err, "tailcall"); 1400 ASSERT_EQ(topts.retval, 1, "tailcall retval"); 1401 1402 i = 0; 1403 err = bpf_map_lookup_elem(main_data_fd, &i, &val); 1404 ASSERT_OK(err, "tailcall count"); 1405 ASSERT_EQ(val, 35, "tailcall count"); 1406 1407 if (test_fentry) { 1408 i = 0; 1409 err = bpf_map_lookup_elem(fentry_data_fd, &i, &val); 1410 ASSERT_OK(err, "fentry count"); 1411 ASSERT_EQ(val, 70, "fentry count"); 1412 } 1413 1414 if (test_fexit) { 1415 i = 0; 1416 err = bpf_map_lookup_elem(fexit_data_fd, &i, &val); 1417 ASSERT_OK(err, "fexit count"); 1418 ASSERT_EQ(val, 70, "fexit count"); 1419 } 1420 1421 out: 1422 bpf_link__destroy(fentry_link); 1423 bpf_link__destroy(fexit_link); 1424 bpf_object__close(fentry_obj); 1425 bpf_object__close(fexit_obj); 1426 bpf_object__close(obj); 1427 } 1428 1429 /* test_tailcall_bpf2bpf_hierarchy_1 checks that the count value of the tail 1430 * call limit enforcement matches with expectations when tailcalls are preceded 1431 * with two bpf2bpf calls. 1432 * 1433 * subprog --tailcall-> entry 1434 * entry < 1435 * subprog --tailcall-> entry 1436 */ 1437 static void test_tailcall_bpf2bpf_hierarchy_1(void) 1438 { 1439 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o", 1440 false, false, false); 1441 } 1442 1443 /* test_tailcall_bpf2bpf_hierarchy_fentry checks that the count value of the 1444 * tail call limit enforcement matches with expectations when tailcalls are 1445 * preceded with two bpf2bpf calls, and the two subprogs are traced by fentry. 1446 */ 1447 static void test_tailcall_bpf2bpf_hierarchy_fentry(void) 1448 { 1449 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o", 1450 true, false, false); 1451 } 1452 1453 /* test_tailcall_bpf2bpf_hierarchy_fexit checks that the count value of the tail 1454 * call limit enforcement matches with expectations when tailcalls are preceded 1455 * with two bpf2bpf calls, and the two subprogs are traced by fexit. 1456 */ 1457 static void test_tailcall_bpf2bpf_hierarchy_fexit(void) 1458 { 1459 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o", 1460 false, true, false); 1461 } 1462 1463 /* test_tailcall_bpf2bpf_hierarchy_fentry_fexit checks that the count value of 1464 * the tail call limit enforcement matches with expectations when tailcalls are 1465 * preceded with two bpf2bpf calls, and the two subprogs are traced by both 1466 * fentry and fexit. 1467 */ 1468 static void test_tailcall_bpf2bpf_hierarchy_fentry_fexit(void) 1469 { 1470 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o", 1471 true, true, false); 1472 } 1473 1474 /* test_tailcall_bpf2bpf_hierarchy_fentry_entry checks that the count value of 1475 * the tail call limit enforcement matches with expectations when tailcalls are 1476 * preceded with two bpf2bpf calls in fentry. 1477 */ 1478 static void test_tailcall_bpf2bpf_hierarchy_fentry_entry(void) 1479 { 1480 test_tailcall_hierarchy_count("tc_dummy.bpf.o", false, false, true); 1481 } 1482 1483 /* test_tailcall_bpf2bpf_hierarchy_2 checks that the count value of the tail 1484 * call limit enforcement matches with expectations: 1485 * 1486 * subprog_tail0 --tailcall-> classifier_0 -> subprog_tail0 1487 * entry < 1488 * subprog_tail1 --tailcall-> classifier_1 -> subprog_tail1 1489 */ 1490 static void test_tailcall_bpf2bpf_hierarchy_2(void) 1491 { 1492 RUN_TESTS(tailcall_bpf2bpf_hierarchy2); 1493 } 1494 1495 /* test_tailcall_bpf2bpf_hierarchy_3 checks that the count value of the tail 1496 * call limit enforcement matches with expectations: 1497 * 1498 * subprog with jmp_table0 to classifier_0 1499 * entry --tailcall-> classifier_0 < 1500 * subprog with jmp_table1 to classifier_0 1501 */ 1502 static void test_tailcall_bpf2bpf_hierarchy_3(void) 1503 { 1504 RUN_TESTS(tailcall_bpf2bpf_hierarchy3); 1505 } 1506 1507 /* test_tailcall_freplace checks that the freplace prog fails to update the 1508 * prog_array map, no matter whether the freplace prog attaches to its target. 1509 */ 1510 static void test_tailcall_freplace(void) 1511 { 1512 struct tailcall_freplace *freplace_skel = NULL; 1513 struct bpf_link *freplace_link = NULL; 1514 struct bpf_program *freplace_prog; 1515 struct tc_bpf2bpf *tc_skel = NULL; 1516 int prog_fd, tc_prog_fd, map_fd; 1517 char buff[128] = {}; 1518 int err, key; 1519 1520 LIBBPF_OPTS(bpf_test_run_opts, topts, 1521 .data_in = buff, 1522 .data_size_in = sizeof(buff), 1523 .repeat = 1, 1524 ); 1525 1526 freplace_skel = tailcall_freplace__open(); 1527 if (!ASSERT_OK_PTR(freplace_skel, "tailcall_freplace__open")) 1528 return; 1529 1530 tc_skel = tc_bpf2bpf__open_and_load(); 1531 if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load")) 1532 goto out; 1533 1534 tc_prog_fd = bpf_program__fd(tc_skel->progs.entry_tc); 1535 freplace_prog = freplace_skel->progs.entry_freplace; 1536 err = bpf_program__set_attach_target(freplace_prog, tc_prog_fd, 1537 "subprog_tc"); 1538 if (!ASSERT_OK(err, "set_attach_target")) 1539 goto out; 1540 1541 err = tailcall_freplace__load(freplace_skel); 1542 if (!ASSERT_OK(err, "tailcall_freplace__load")) 1543 goto out; 1544 1545 map_fd = bpf_map__fd(freplace_skel->maps.jmp_table); 1546 prog_fd = bpf_program__fd(freplace_prog); 1547 key = 0; 1548 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY); 1549 ASSERT_ERR(err, "update jmp_table failure"); 1550 1551 freplace_link = bpf_program__attach_freplace(freplace_prog, tc_prog_fd, 1552 "subprog_tc"); 1553 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace")) 1554 goto out; 1555 1556 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY); 1557 ASSERT_ERR(err, "update jmp_table failure"); 1558 1559 out: 1560 bpf_link__destroy(freplace_link); 1561 tailcall_freplace__destroy(freplace_skel); 1562 tc_bpf2bpf__destroy(tc_skel); 1563 } 1564 1565 /* test_tailcall_bpf2bpf_freplace checks the failure that fails to attach a tail 1566 * callee prog with freplace prog or fails to update an extended prog to 1567 * prog_array map. 1568 */ 1569 static void test_tailcall_bpf2bpf_freplace(void) 1570 { 1571 struct tailcall_freplace *freplace_skel = NULL; 1572 struct bpf_link *freplace_link = NULL; 1573 struct tc_bpf2bpf *tc_skel = NULL; 1574 char buff[128] = {}; 1575 int prog_fd, map_fd; 1576 int err, key; 1577 1578 LIBBPF_OPTS(bpf_test_run_opts, topts, 1579 .data_in = buff, 1580 .data_size_in = sizeof(buff), 1581 .repeat = 1, 1582 ); 1583 1584 tc_skel = tc_bpf2bpf__open_and_load(); 1585 if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load")) 1586 goto out; 1587 1588 prog_fd = bpf_program__fd(tc_skel->progs.entry_tc); 1589 freplace_skel = tailcall_freplace__open(); 1590 if (!ASSERT_OK_PTR(freplace_skel, "tailcall_freplace__open")) 1591 goto out; 1592 1593 err = bpf_program__set_attach_target(freplace_skel->progs.entry_freplace, 1594 prog_fd, "subprog_tc"); 1595 if (!ASSERT_OK(err, "set_attach_target")) 1596 goto out; 1597 1598 err = tailcall_freplace__load(freplace_skel); 1599 if (!ASSERT_OK(err, "tailcall_freplace__load")) 1600 goto out; 1601 1602 /* OK to attach then detach freplace prog. */ 1603 1604 freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace, 1605 prog_fd, "subprog_tc"); 1606 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace")) 1607 goto out; 1608 1609 err = bpf_link__destroy(freplace_link); 1610 freplace_link = NULL; 1611 if (!ASSERT_OK(err, "destroy link")) 1612 goto out; 1613 1614 /* OK to update prog_array map then delete element from the map. */ 1615 1616 key = 0; 1617 map_fd = bpf_map__fd(freplace_skel->maps.jmp_table); 1618 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY); 1619 if (!ASSERT_OK(err, "update jmp_table")) 1620 goto out; 1621 1622 err = bpf_map_delete_elem(map_fd, &key); 1623 if (!ASSERT_OK(err, "delete_elem from jmp_table")) 1624 goto out; 1625 1626 /* Fail to attach a tail callee prog with freplace prog. */ 1627 1628 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY); 1629 if (!ASSERT_OK(err, "update jmp_table")) 1630 goto out; 1631 1632 freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace, 1633 prog_fd, "subprog_tc"); 1634 if (!ASSERT_ERR_PTR(freplace_link, "attach_freplace failure")) 1635 goto out; 1636 1637 err = bpf_map_delete_elem(map_fd, &key); 1638 if (!ASSERT_OK(err, "delete_elem from jmp_table")) 1639 goto out; 1640 1641 /* Fail to update an extended prog to prog_array map. */ 1642 1643 freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace, 1644 prog_fd, "subprog_tc"); 1645 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace")) 1646 goto out; 1647 1648 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY); 1649 if (!ASSERT_ERR(err, "update jmp_table failure")) 1650 goto out; 1651 1652 out: 1653 bpf_link__destroy(freplace_link); 1654 tailcall_freplace__destroy(freplace_skel); 1655 tc_bpf2bpf__destroy(tc_skel); 1656 } 1657 1658 static void test_tailcall_failure() 1659 { 1660 RUN_TESTS(tailcall_fail); 1661 } 1662 1663 static void test_tailcall_cgrp_storage(void) 1664 { 1665 struct tailcall_cgrp_storage_owner *owner_skel = NULL; 1666 struct tailcall_cgrp_storage *skel = NULL; 1667 int err, key = 0, prog_array_fd, prog_fd, storage_map_fd; 1668 1669 owner_skel = tailcall_cgrp_storage_owner__open_and_load(); 1670 if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) 1671 return; 1672 1673 prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); 1674 storage_map_fd = bpf_map__fd(owner_skel->maps.storage_map); 1675 1676 skel = tailcall_cgrp_storage__open(); 1677 if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage__open")) 1678 goto out; 1679 1680 err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd); 1681 if (!ASSERT_OK(err, "reuse_prog_array")) 1682 goto out; 1683 1684 err = bpf_map__reuse_fd(skel->maps.storage_map, storage_map_fd); 1685 if (!ASSERT_OK(err, "reuse_storage_map")) 1686 goto out; 1687 1688 err = bpf_object__load(skel->obj); 1689 if (!ASSERT_OK(err, "tailcall_cgrp_storage__load")) 1690 goto out; 1691 1692 prog_fd = bpf_program__fd(skel->progs.callee_prog); 1693 err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); 1694 ASSERT_OK(err, "update_prog_array"); 1695 out: 1696 tailcall_cgrp_storage__destroy(skel); 1697 tailcall_cgrp_storage_owner__destroy(owner_skel); 1698 } 1699 1700 static void test_tailcall_cgrp_storage_diff_storage(void) 1701 { 1702 struct tailcall_cgrp_storage_owner *owner_skel = NULL; 1703 struct tailcall_cgrp_storage *skel = NULL; 1704 int err, prog_array_fd; 1705 1706 owner_skel = tailcall_cgrp_storage_owner__open_and_load(); 1707 if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) 1708 return; 1709 1710 prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); 1711 1712 skel = tailcall_cgrp_storage__open(); 1713 if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage__open")) 1714 goto out; 1715 1716 err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd); 1717 if (!ASSERT_OK(err, "reuse_prog_array")) 1718 goto out; 1719 1720 err = bpf_object__load(skel->obj); 1721 ASSERT_ERR(err, "tailcall_cgrp_storage__load"); 1722 out: 1723 tailcall_cgrp_storage__destroy(skel); 1724 tailcall_cgrp_storage_owner__destroy(owner_skel); 1725 } 1726 1727 static void test_tailcall_cgrp_storage_no_storage(void) 1728 { 1729 struct tailcall_cgrp_storage_owner *owner_skel = NULL; 1730 struct tailcall_cgrp_storage_no_storage *skel = NULL; 1731 int err, prog_array_fd; 1732 1733 owner_skel = tailcall_cgrp_storage_owner__open_and_load(); 1734 if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) 1735 return; 1736 1737 prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); 1738 1739 skel = tailcall_cgrp_storage_no_storage__open(); 1740 if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage_no_storage__open")) 1741 goto out; 1742 1743 err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd); 1744 if (!ASSERT_OK(err, "reuse_prog_array")) 1745 goto out; 1746 1747 err = bpf_object__load(skel->obj); 1748 ASSERT_ERR(err, "tailcall_cgrp_storage_no_storage__load"); 1749 out: 1750 tailcall_cgrp_storage_no_storage__destroy(skel); 1751 tailcall_cgrp_storage_owner__destroy(owner_skel); 1752 } 1753 1754 static void test_tailcall_cgrp_storage_no_storage_leaf(void) 1755 { 1756 struct tailcall_cgrp_storage_owner *owner_skel = NULL; 1757 struct tailcall_cgrp_storage_no_storage *skel = NULL; 1758 int err, key = 0, prog_array_fd, prog_fd; 1759 1760 owner_skel = tailcall_cgrp_storage_owner__open_and_load(); 1761 if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) 1762 return; 1763 1764 prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); 1765 1766 skel = tailcall_cgrp_storage_no_storage__open_and_load(); 1767 if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage_no_storage__open_and_load")) 1768 goto out; 1769 1770 prog_fd = bpf_program__fd(skel->progs.leaf_prog); 1771 err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); 1772 if (!ASSERT_OK(err, "update_prog_array_leaf")) 1773 goto out; 1774 1775 prog_fd = bpf_program__fd(skel->progs.caller_prog); 1776 err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); 1777 ASSERT_ERR(err, "update_prog_array_bridge"); 1778 out: 1779 tailcall_cgrp_storage_no_storage__destroy(skel); 1780 tailcall_cgrp_storage_owner__destroy(owner_skel); 1781 } 1782 1783 static void test_tailcall_cgrp_storage_no_storage_bridge(void) 1784 { 1785 struct tailcall_cgrp_storage_owner *owner_skel = NULL; 1786 struct tailcall_cgrp_storage_no_storage *bridge_skel = NULL; 1787 struct tailcall_cgrp_storage *callee_skel = NULL; 1788 int err, key = 0, prog_array_fd, prog_fd, storage_map_fd; 1789 1790 owner_skel = tailcall_cgrp_storage_owner__open_and_load(); 1791 if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) 1792 return; 1793 1794 prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); 1795 storage_map_fd = bpf_map__fd(owner_skel->maps.storage_map); 1796 1797 callee_skel = tailcall_cgrp_storage__open(); 1798 if (!ASSERT_OK_PTR(callee_skel, "tailcall_cgrp_storage__open")) 1799 goto out; 1800 1801 bpf_program__set_autoload(callee_skel->progs.caller_prog, false); 1802 1803 err = bpf_map__reuse_fd(callee_skel->maps.prog_array, prog_array_fd); 1804 if (!ASSERT_OK(err, "reuse_prog_array")) 1805 goto out; 1806 1807 err = bpf_map__reuse_fd(callee_skel->maps.storage_map, storage_map_fd); 1808 if (!ASSERT_OK(err, "reuse_storage_map")) 1809 goto out; 1810 1811 err = bpf_object__load(callee_skel->obj); 1812 if (!ASSERT_OK(err, "tailcall_cgrp_storage__load")) 1813 goto out; 1814 1815 prog_fd = bpf_program__fd(callee_skel->progs.callee_prog); 1816 err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); 1817 if (!ASSERT_OK(err, "update_prog_array")) 1818 goto out; 1819 1820 bridge_skel = tailcall_cgrp_storage_no_storage__open(); 1821 if (!ASSERT_OK_PTR(bridge_skel, "tailcall_cgrp_storage_no_storage__open")) 1822 goto out; 1823 1824 err = bpf_map__reuse_fd(bridge_skel->maps.prog_array, prog_array_fd); 1825 if (!ASSERT_OK(err, "reuse_prog_array")) 1826 goto out; 1827 1828 err = bpf_object__load(bridge_skel->obj); 1829 ASSERT_ERR(err, "tailcall_cgrp_storage_no_storage_bridge__load"); 1830 out: 1831 tailcall_cgrp_storage_no_storage__destroy(bridge_skel); 1832 tailcall_cgrp_storage__destroy(callee_skel); 1833 tailcall_cgrp_storage_owner__destroy(owner_skel); 1834 } 1835 1836 noinline void uprobe_sleepable_trigger(void) 1837 { 1838 asm volatile (""); 1839 } 1840 1841 static void test_tailcall_sleepable(void) 1842 { 1843 LIBBPF_OPTS(bpf_uprobe_opts, opts); 1844 struct tailcall_sleepable *skel; 1845 int prog_fd, map_fd; 1846 int err, key; 1847 1848 skel = tailcall_sleepable__open(); 1849 if (!ASSERT_OK_PTR(skel, "tailcall_sleepable__open")) 1850 return; 1851 1852 /* 1853 * Test that we can't load uprobe_normal and uprobe_sleepable_1, 1854 * because they share tailcall map. 1855 */ 1856 bpf_program__set_autoload(skel->progs.uprobe_normal, true); 1857 bpf_program__set_autoload(skel->progs.uprobe_sleepable_1, true); 1858 1859 err = tailcall_sleepable__load(skel); 1860 if (!ASSERT_ERR(err, "tailcall_sleepable__load")) 1861 goto out; 1862 1863 tailcall_sleepable__destroy(skel); 1864 1865 /* 1866 * Test that we can tail call from sleepable to sleepable program. 1867 */ 1868 skel = tailcall_sleepable__open(); 1869 if (!ASSERT_OK_PTR(skel, "tailcall_sleepable__open")) 1870 return; 1871 1872 bpf_program__set_autoload(skel->progs.uprobe_sleepable_1, true); 1873 bpf_program__set_autoload(skel->progs.uprobe_sleepable_2, true); 1874 1875 err = tailcall_sleepable__load(skel); 1876 if (!ASSERT_OK(err, "tailcall_sleepable__load")) 1877 goto out; 1878 1879 /* Add sleepable uprobe_sleepable_2 to jmp_table[0]. */ 1880 key = 0; 1881 prog_fd = bpf_program__fd(skel->progs.uprobe_sleepable_2); 1882 map_fd = bpf_map__fd(skel->maps.jmp_table); 1883 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY); 1884 if (!ASSERT_OK(err, "update jmp_table")) 1885 goto out; 1886 1887 skel->bss->my_pid = getpid(); 1888 1889 /* Attach uprobe_sleepable_1 to uprobe_sleepable_trigger and hit it. */ 1890 opts.func_name = "uprobe_sleepable_trigger"; 1891 skel->links.uprobe_sleepable_1 = bpf_program__attach_uprobe_opts( 1892 skel->progs.uprobe_sleepable_1, 1893 -1, 1894 "/proc/self/exe", 1895 0 /* offset */, 1896 &opts); 1897 if (!ASSERT_OK_PTR(skel->links.uprobe_sleepable_1, "bpf_program__attach_uprobe_opts")) 1898 goto out; 1899 1900 uprobe_sleepable_trigger(); 1901 ASSERT_EQ(skel->bss->executed, 1, "executed"); 1902 1903 out: 1904 tailcall_sleepable__destroy(skel); 1905 } 1906 1907 static void test_tailcall_callback(void) 1908 { 1909 RUN_TESTS(tailcall_callback); 1910 } 1911 1912 static void test_tailcall_bpf2bpf_fexit_links(void) 1913 { 1914 struct tailcall_bpf2bpf_fexit *skel1 = NULL, *skel2 = NULL; 1915 struct tailcall_bpf2bpf2 *skel_tc; 1916 int err, prog_fd; 1917 1918 skel_tc = tailcall_bpf2bpf2__open_and_load(); 1919 if (!ASSERT_OK_PTR(skel_tc, "tailcall_bpf2bpf2__open_and_load")) 1920 return; 1921 1922 skel1 = tailcall_bpf2bpf_fexit__open(); 1923 if (!ASSERT_OK_PTR(skel1, "tailcall_bpf2bpf_fexit__open")) 1924 goto out; 1925 1926 prog_fd = bpf_program__fd(skel_tc->progs.classifier_0); 1927 err = bpf_program__set_attach_target(skel1->progs.fexit, prog_fd, "subprog_tail"); 1928 if (!ASSERT_OK(err, "bpf_program__set_attach_target")) 1929 goto out; 1930 1931 err = tailcall_bpf2bpf_fexit__load(skel1); 1932 if (!ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load")) 1933 goto out; 1934 1935 skel1->links.fexit = bpf_program__attach_trace(skel1->progs.fexit); 1936 if (!ASSERT_OK_PTR(skel1->links.fexit, "bpf_program__attach_trace")) 1937 goto out; 1938 1939 skel2 = tailcall_bpf2bpf_fexit__open(); 1940 if (!ASSERT_OK_PTR(skel2, "tailcall_bpf2bpf_fexit__open")) 1941 goto out; 1942 1943 err = bpf_program__set_attach_target(skel2->progs.fexit, prog_fd, "subprog_tail"); 1944 if (!ASSERT_OK(err, "bpf_program__set_attach_target")) 1945 goto out; 1946 1947 err = tailcall_bpf2bpf_fexit__load(skel2); 1948 ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load"); 1949 1950 out: 1951 tailcall_bpf2bpf_fexit__destroy(skel1); 1952 tailcall_bpf2bpf_fexit__destroy(skel2); 1953 tailcall_bpf2bpf2__destroy(skel_tc); 1954 } 1955 1956 void test_tailcalls(void) 1957 { 1958 if (test__start_subtest("tailcall_1")) 1959 test_tailcall_1(); 1960 if (test__start_subtest("tailcall_2")) 1961 test_tailcall_2(); 1962 if (test__start_subtest("tailcall_3")) 1963 test_tailcall_3(); 1964 if (test__start_subtest("tailcall_4")) 1965 test_tailcall_4(); 1966 if (test__start_subtest("tailcall_5")) 1967 test_tailcall_5(); 1968 if (test__start_subtest("tailcall_6")) 1969 test_tailcall_6(); 1970 if (test__start_subtest("tailcall_bpf2bpf_1")) 1971 test_tailcall_bpf2bpf_1(); 1972 if (test__start_subtest("tailcall_bpf2bpf_2")) 1973 test_tailcall_bpf2bpf_2(); 1974 if (test__start_subtest("tailcall_bpf2bpf_3")) 1975 test_tailcall_bpf2bpf_3(); 1976 if (test__start_subtest("tailcall_bpf2bpf_4")) 1977 test_tailcall_bpf2bpf_4(false); 1978 if (test__start_subtest("tailcall_bpf2bpf_5")) 1979 test_tailcall_bpf2bpf_4(true); 1980 if (test__start_subtest("tailcall_bpf2bpf_6")) 1981 test_tailcall_bpf2bpf_6(); 1982 if (test__start_subtest("tailcall_bpf2bpf_fentry")) 1983 test_tailcall_bpf2bpf_fentry(); 1984 if (test__start_subtest("tailcall_bpf2bpf_fexit")) 1985 test_tailcall_bpf2bpf_fexit(); 1986 if (test__start_subtest("tailcall_bpf2bpf_fentry_fexit")) 1987 test_tailcall_bpf2bpf_fentry_fexit(); 1988 if (test__start_subtest("tailcall_bpf2bpf_fentry_entry")) 1989 test_tailcall_bpf2bpf_fentry_entry(); 1990 if (test__start_subtest("tailcall_poke")) 1991 test_tailcall_poke(); 1992 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_1")) 1993 test_tailcall_bpf2bpf_hierarchy_1(); 1994 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fentry")) 1995 test_tailcall_bpf2bpf_hierarchy_fentry(); 1996 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fexit")) 1997 test_tailcall_bpf2bpf_hierarchy_fexit(); 1998 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fentry_fexit")) 1999 test_tailcall_bpf2bpf_hierarchy_fentry_fexit(); 2000 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fentry_entry")) 2001 test_tailcall_bpf2bpf_hierarchy_fentry_entry(); 2002 test_tailcall_bpf2bpf_hierarchy_2(); 2003 test_tailcall_bpf2bpf_hierarchy_3(); 2004 if (test__start_subtest("tailcall_freplace")) 2005 test_tailcall_freplace(); 2006 if (test__start_subtest("tailcall_bpf2bpf_freplace")) 2007 test_tailcall_bpf2bpf_freplace(); 2008 if (test__start_subtest("tailcall_failure")) 2009 test_tailcall_failure(); 2010 if (test__start_subtest("tailcall_sleepable")) 2011 test_tailcall_sleepable(); 2012 if (test__start_subtest("tailcall_cgrp_storage")) 2013 test_tailcall_cgrp_storage(); 2014 if (test__start_subtest("tailcall_cgrp_storage_diff_storage")) 2015 test_tailcall_cgrp_storage_diff_storage(); 2016 if (test__start_subtest("tailcall_cgrp_storage_no_storage")) 2017 test_tailcall_cgrp_storage_no_storage(); 2018 if (test__start_subtest("tailcall_cgrp_storage_no_storage_leaf")) 2019 test_tailcall_cgrp_storage_no_storage_leaf(); 2020 if (test__start_subtest("tailcall_cgrp_storage_no_storage_bridge")) 2021 test_tailcall_cgrp_storage_no_storage_bridge(); 2022 test_tailcall_callback(); 2023 if (test__start_subtest("tailcall_bpf2bpf_fexit_links")) 2024 test_tailcall_bpf2bpf_fexit_links(); 2025 } 2026