1 // SPDX-License-Identifier: GPL-2.0
2 #include <unistd.h>
3 #include <test_progs.h>
4 #include <network_helpers.h>
5 #include "tailcall_poke.skel.h"
6 #include "tailcall_bpf2bpf_hierarchy2.skel.h"
7 #include "tailcall_bpf2bpf_hierarchy3.skel.h"
8 #include "tailcall_freplace.skel.h"
9 #include "tc_bpf2bpf.skel.h"
10 #include "tailcall_fail.skel.h"
11 #include "tailcall_cgrp_storage_owner.skel.h"
12 #include "tailcall_cgrp_storage_no_storage.skel.h"
13 #include "tailcall_cgrp_storage.skel.h"
14 #include "tailcall_sleepable.skel.h"
15 #include "tailcall_callback.skel.h"
16 #include "tailcall_bpf2bpf2.skel.h"
17 #include "tailcall_bpf2bpf_fexit.skel.h"
18
19 /* test_tailcall_1 checks basic functionality by patching multiple locations
20 * in a single program for a single tail call slot with nop->jmp, jmp->nop
21 * and jmp->jmp rewrites. Also checks for nop->nop.
22 */
test_tailcall_1(void)23 static void test_tailcall_1(void)
24 {
25 int err, map_fd, prog_fd, main_fd, i, j;
26 struct bpf_map *prog_array;
27 struct bpf_program *prog;
28 struct bpf_object *obj;
29 char prog_name[32];
30 char buff[128] = {};
31 LIBBPF_OPTS(bpf_test_run_opts, topts,
32 .data_in = buff,
33 .data_size_in = sizeof(buff),
34 .repeat = 1,
35 );
36
37 err = bpf_prog_test_load("tailcall1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
38 &prog_fd);
39 if (CHECK_FAIL(err))
40 return;
41
42 prog = bpf_object__find_program_by_name(obj, "entry");
43 if (CHECK_FAIL(!prog))
44 goto out;
45
46 main_fd = bpf_program__fd(prog);
47 if (CHECK_FAIL(main_fd < 0))
48 goto out;
49
50 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
51 if (CHECK_FAIL(!prog_array))
52 goto out;
53
54 map_fd = bpf_map__fd(prog_array);
55 if (CHECK_FAIL(map_fd < 0))
56 goto out;
57
58 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
59 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
60
61 prog = bpf_object__find_program_by_name(obj, prog_name);
62 if (CHECK_FAIL(!prog))
63 goto out;
64
65 prog_fd = bpf_program__fd(prog);
66 if (CHECK_FAIL(prog_fd < 0))
67 goto out;
68
69 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
70 if (CHECK_FAIL(err))
71 goto out;
72 }
73
74 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
75 err = bpf_prog_test_run_opts(main_fd, &topts);
76 ASSERT_OK(err, "tailcall");
77 ASSERT_EQ(topts.retval, i, "tailcall retval");
78
79 err = bpf_map_delete_elem(map_fd, &i);
80 if (CHECK_FAIL(err))
81 goto out;
82 }
83
84 err = bpf_prog_test_run_opts(main_fd, &topts);
85 ASSERT_OK(err, "tailcall");
86 ASSERT_EQ(topts.retval, 3, "tailcall retval");
87
88 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
89 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
90
91 prog = bpf_object__find_program_by_name(obj, prog_name);
92 if (CHECK_FAIL(!prog))
93 goto out;
94
95 prog_fd = bpf_program__fd(prog);
96 if (CHECK_FAIL(prog_fd < 0))
97 goto out;
98
99 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
100 if (CHECK_FAIL(err))
101 goto out;
102 }
103
104 err = bpf_prog_test_run_opts(main_fd, &topts);
105 ASSERT_OK(err, "tailcall");
106 ASSERT_OK(topts.retval, "tailcall retval");
107
108 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
109 j = bpf_map__max_entries(prog_array) - 1 - i;
110 snprintf(prog_name, sizeof(prog_name), "classifier_%d", j);
111
112 prog = bpf_object__find_program_by_name(obj, prog_name);
113 if (CHECK_FAIL(!prog))
114 goto out;
115
116 prog_fd = bpf_program__fd(prog);
117 if (CHECK_FAIL(prog_fd < 0))
118 goto out;
119
120 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
121 if (CHECK_FAIL(err))
122 goto out;
123 }
124
125 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
126 j = bpf_map__max_entries(prog_array) - 1 - i;
127
128 err = bpf_prog_test_run_opts(main_fd, &topts);
129 ASSERT_OK(err, "tailcall");
130 ASSERT_EQ(topts.retval, j, "tailcall retval");
131
132 err = bpf_map_delete_elem(map_fd, &i);
133 if (CHECK_FAIL(err))
134 goto out;
135 }
136
137 err = bpf_prog_test_run_opts(main_fd, &topts);
138 ASSERT_OK(err, "tailcall");
139 ASSERT_EQ(topts.retval, 3, "tailcall retval");
140
141 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
142 err = bpf_map_delete_elem(map_fd, &i);
143 if (CHECK_FAIL(err >= 0 || errno != ENOENT))
144 goto out;
145
146 err = bpf_prog_test_run_opts(main_fd, &topts);
147 ASSERT_OK(err, "tailcall");
148 ASSERT_EQ(topts.retval, 3, "tailcall retval");
149 }
150
151 out:
152 bpf_object__close(obj);
153 }
154
155 /* test_tailcall_2 checks that patching multiple programs for a single
156 * tail call slot works. It also jumps through several programs and tests
157 * the tail call limit counter.
158 */
test_tailcall_2(void)159 static void test_tailcall_2(void)
160 {
161 int err, map_fd, prog_fd, main_fd, i;
162 struct bpf_map *prog_array;
163 struct bpf_program *prog;
164 struct bpf_object *obj;
165 char prog_name[32];
166 char buff[128] = {};
167 LIBBPF_OPTS(bpf_test_run_opts, topts,
168 .data_in = buff,
169 .data_size_in = sizeof(buff),
170 .repeat = 1,
171 );
172
173 err = bpf_prog_test_load("tailcall2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
174 &prog_fd);
175 if (CHECK_FAIL(err))
176 return;
177
178 prog = bpf_object__find_program_by_name(obj, "entry");
179 if (CHECK_FAIL(!prog))
180 goto out;
181
182 main_fd = bpf_program__fd(prog);
183 if (CHECK_FAIL(main_fd < 0))
184 goto out;
185
186 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
187 if (CHECK_FAIL(!prog_array))
188 goto out;
189
190 map_fd = bpf_map__fd(prog_array);
191 if (CHECK_FAIL(map_fd < 0))
192 goto out;
193
194 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
195 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
196
197 prog = bpf_object__find_program_by_name(obj, prog_name);
198 if (CHECK_FAIL(!prog))
199 goto out;
200
201 prog_fd = bpf_program__fd(prog);
202 if (CHECK_FAIL(prog_fd < 0))
203 goto out;
204
205 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
206 if (CHECK_FAIL(err))
207 goto out;
208 }
209
210 err = bpf_prog_test_run_opts(main_fd, &topts);
211 ASSERT_OK(err, "tailcall");
212 ASSERT_EQ(topts.retval, 2, "tailcall retval");
213
214 i = 2;
215 err = bpf_map_delete_elem(map_fd, &i);
216 if (CHECK_FAIL(err))
217 goto out;
218
219 err = bpf_prog_test_run_opts(main_fd, &topts);
220 ASSERT_OK(err, "tailcall");
221 ASSERT_EQ(topts.retval, 1, "tailcall retval");
222
223 i = 0;
224 err = bpf_map_delete_elem(map_fd, &i);
225 if (CHECK_FAIL(err))
226 goto out;
227
228 err = bpf_prog_test_run_opts(main_fd, &topts);
229 ASSERT_OK(err, "tailcall");
230 ASSERT_EQ(topts.retval, 3, "tailcall retval");
231 out:
232 bpf_object__close(obj);
233 }
234
test_tailcall_count(const char * which,bool test_fentry,bool test_fexit)235 static void test_tailcall_count(const char *which, bool test_fentry,
236 bool test_fexit)
237 {
238 struct bpf_object *obj = NULL, *fentry_obj = NULL, *fexit_obj = NULL;
239 struct bpf_link *fentry_link = NULL, *fexit_link = NULL;
240 int err, map_fd, prog_fd, main_fd, data_fd, i, val;
241 struct bpf_map *prog_array, *data_map;
242 struct bpf_program *prog;
243 char buff[128] = {};
244 LIBBPF_OPTS(bpf_test_run_opts, topts,
245 .data_in = buff,
246 .data_size_in = sizeof(buff),
247 .repeat = 1,
248 );
249
250 err = bpf_prog_test_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj,
251 &prog_fd);
252 if (CHECK_FAIL(err))
253 return;
254
255 prog = bpf_object__find_program_by_name(obj, "entry");
256 if (CHECK_FAIL(!prog))
257 goto out;
258
259 main_fd = bpf_program__fd(prog);
260 if (CHECK_FAIL(main_fd < 0))
261 goto out;
262
263 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
264 if (CHECK_FAIL(!prog_array))
265 goto out;
266
267 map_fd = bpf_map__fd(prog_array);
268 if (CHECK_FAIL(map_fd < 0))
269 goto out;
270
271 prog = bpf_object__find_program_by_name(obj, "classifier_0");
272 if (CHECK_FAIL(!prog))
273 goto out;
274
275 prog_fd = bpf_program__fd(prog);
276 if (CHECK_FAIL(prog_fd < 0))
277 goto out;
278
279 i = 0;
280 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
281 if (CHECK_FAIL(err))
282 goto out;
283
284 if (test_fentry) {
285 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o",
286 NULL);
287 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file"))
288 goto out;
289
290 prog = bpf_object__find_program_by_name(fentry_obj, "fentry");
291 if (!ASSERT_OK_PTR(prog, "find fentry prog"))
292 goto out;
293
294 err = bpf_program__set_attach_target(prog, prog_fd,
295 "subprog_tail");
296 if (!ASSERT_OK(err, "set_attach_target subprog_tail"))
297 goto out;
298
299 err = bpf_object__load(fentry_obj);
300 if (!ASSERT_OK(err, "load fentry_obj"))
301 goto out;
302
303 fentry_link = bpf_program__attach_trace(prog);
304 if (!ASSERT_OK_PTR(fentry_link, "attach_trace"))
305 goto out;
306 }
307
308 if (test_fexit) {
309 fexit_obj = bpf_object__open_file("tailcall_bpf2bpf_fexit.bpf.o",
310 NULL);
311 if (!ASSERT_OK_PTR(fexit_obj, "open fexit_obj file"))
312 goto out;
313
314 prog = bpf_object__find_program_by_name(fexit_obj, "fexit");
315 if (!ASSERT_OK_PTR(prog, "find fexit prog"))
316 goto out;
317
318 err = bpf_program__set_attach_target(prog, prog_fd,
319 "subprog_tail");
320 if (!ASSERT_OK(err, "set_attach_target subprog_tail"))
321 goto out;
322
323 err = bpf_object__load(fexit_obj);
324 if (!ASSERT_OK(err, "load fexit_obj"))
325 goto out;
326
327 fexit_link = bpf_program__attach_trace(prog);
328 if (!ASSERT_OK_PTR(fexit_link, "attach_trace"))
329 goto out;
330 }
331
332 err = bpf_prog_test_run_opts(main_fd, &topts);
333 ASSERT_OK(err, "tailcall");
334 ASSERT_EQ(topts.retval, 1, "tailcall retval");
335
336 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
337 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
338 goto out;
339
340 data_fd = bpf_map__fd(data_map);
341 if (CHECK_FAIL(data_fd < 0))
342 goto out;
343
344 i = 0;
345 err = bpf_map_lookup_elem(data_fd, &i, &val);
346 ASSERT_OK(err, "tailcall count");
347 ASSERT_EQ(val, 33, "tailcall count");
348
349 if (test_fentry) {
350 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss");
351 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),
352 "find tailcall_bpf2bpf_fentry.bss map"))
353 goto out;
354
355 data_fd = bpf_map__fd(data_map);
356 if (!ASSERT_FALSE(data_fd < 0,
357 "find tailcall_bpf2bpf_fentry.bss map fd"))
358 goto out;
359
360 i = 0;
361 err = bpf_map_lookup_elem(data_fd, &i, &val);
362 ASSERT_OK(err, "fentry count");
363 ASSERT_EQ(val, 33, "fentry count");
364 }
365
366 if (test_fexit) {
367 data_map = bpf_object__find_map_by_name(fexit_obj, ".bss");
368 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),
369 "find tailcall_bpf2bpf_fexit.bss map"))
370 goto out;
371
372 data_fd = bpf_map__fd(data_map);
373 if (!ASSERT_FALSE(data_fd < 0,
374 "find tailcall_bpf2bpf_fexit.bss map fd"))
375 goto out;
376
377 i = 0;
378 err = bpf_map_lookup_elem(data_fd, &i, &val);
379 ASSERT_OK(err, "fexit count");
380 ASSERT_EQ(val, 33, "fexit count");
381 }
382
383 i = 0;
384 err = bpf_map_delete_elem(map_fd, &i);
385 if (CHECK_FAIL(err))
386 goto out;
387
388 err = bpf_prog_test_run_opts(main_fd, &topts);
389 ASSERT_OK(err, "tailcall");
390 ASSERT_OK(topts.retval, "tailcall retval");
391 out:
392 bpf_link__destroy(fentry_link);
393 bpf_link__destroy(fexit_link);
394 bpf_object__close(fentry_obj);
395 bpf_object__close(fexit_obj);
396 bpf_object__close(obj);
397 }
398
399 /* test_tailcall_3 checks that the count value of the tail call limit
400 * enforcement matches with expectations. JIT uses direct jump.
401 */
test_tailcall_3(void)402 static void test_tailcall_3(void)
403 {
404 test_tailcall_count("tailcall3.bpf.o", false, false);
405 }
406
407 /* test_tailcall_6 checks that the count value of the tail call limit
408 * enforcement matches with expectations. JIT uses indirect jump.
409 */
test_tailcall_6(void)410 static void test_tailcall_6(void)
411 {
412 test_tailcall_count("tailcall6.bpf.o", false, false);
413 }
414
415 /* test_tailcall_4 checks that the kernel properly selects indirect jump
416 * for the case where the key is not known. Latter is passed via global
417 * data to select different targets we can compare return value of.
418 */
test_tailcall_4(void)419 static void test_tailcall_4(void)
420 {
421 int err, map_fd, prog_fd, main_fd, data_fd, i;
422 struct bpf_map *prog_array, *data_map;
423 struct bpf_program *prog;
424 struct bpf_object *obj;
425 static const int zero = 0;
426 char buff[128] = {};
427 char prog_name[32];
428 LIBBPF_OPTS(bpf_test_run_opts, topts,
429 .data_in = buff,
430 .data_size_in = sizeof(buff),
431 .repeat = 1,
432 );
433
434 err = bpf_prog_test_load("tailcall4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
435 &prog_fd);
436 if (CHECK_FAIL(err))
437 return;
438
439 prog = bpf_object__find_program_by_name(obj, "entry");
440 if (CHECK_FAIL(!prog))
441 goto out;
442
443 main_fd = bpf_program__fd(prog);
444 if (CHECK_FAIL(main_fd < 0))
445 goto out;
446
447 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
448 if (CHECK_FAIL(!prog_array))
449 goto out;
450
451 map_fd = bpf_map__fd(prog_array);
452 if (CHECK_FAIL(map_fd < 0))
453 goto out;
454
455 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
456 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
457 goto out;
458
459 data_fd = bpf_map__fd(data_map);
460 if (CHECK_FAIL(data_fd < 0))
461 goto out;
462
463 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
464 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
465
466 prog = bpf_object__find_program_by_name(obj, prog_name);
467 if (CHECK_FAIL(!prog))
468 goto out;
469
470 prog_fd = bpf_program__fd(prog);
471 if (CHECK_FAIL(prog_fd < 0))
472 goto out;
473
474 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
475 if (CHECK_FAIL(err))
476 goto out;
477 }
478
479 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
480 err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);
481 if (CHECK_FAIL(err))
482 goto out;
483
484 err = bpf_prog_test_run_opts(main_fd, &topts);
485 ASSERT_OK(err, "tailcall");
486 ASSERT_EQ(topts.retval, i, "tailcall retval");
487 }
488
489 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
490 err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);
491 if (CHECK_FAIL(err))
492 goto out;
493
494 err = bpf_map_delete_elem(map_fd, &i);
495 if (CHECK_FAIL(err))
496 goto out;
497
498 err = bpf_prog_test_run_opts(main_fd, &topts);
499 ASSERT_OK(err, "tailcall");
500 ASSERT_EQ(topts.retval, 3, "tailcall retval");
501 }
502 out:
503 bpf_object__close(obj);
504 }
505
506 /* test_tailcall_5 probes similarly to test_tailcall_4 that the kernel generates
507 * an indirect jump when the keys are const but different from different branches.
508 */
test_tailcall_5(void)509 static void test_tailcall_5(void)
510 {
511 int err, map_fd, prog_fd, main_fd, data_fd, i, key[] = { 1111, 1234, 5678 };
512 struct bpf_map *prog_array, *data_map;
513 struct bpf_program *prog;
514 struct bpf_object *obj;
515 static const int zero = 0;
516 char buff[128] = {};
517 char prog_name[32];
518 LIBBPF_OPTS(bpf_test_run_opts, topts,
519 .data_in = buff,
520 .data_size_in = sizeof(buff),
521 .repeat = 1,
522 );
523
524 err = bpf_prog_test_load("tailcall5.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
525 &prog_fd);
526 if (CHECK_FAIL(err))
527 return;
528
529 prog = bpf_object__find_program_by_name(obj, "entry");
530 if (CHECK_FAIL(!prog))
531 goto out;
532
533 main_fd = bpf_program__fd(prog);
534 if (CHECK_FAIL(main_fd < 0))
535 goto out;
536
537 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
538 if (CHECK_FAIL(!prog_array))
539 goto out;
540
541 map_fd = bpf_map__fd(prog_array);
542 if (CHECK_FAIL(map_fd < 0))
543 goto out;
544
545 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
546 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
547 goto out;
548
549 data_fd = bpf_map__fd(data_map);
550 if (CHECK_FAIL(data_fd < 0))
551 goto out;
552
553 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
554 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
555
556 prog = bpf_object__find_program_by_name(obj, prog_name);
557 if (CHECK_FAIL(!prog))
558 goto out;
559
560 prog_fd = bpf_program__fd(prog);
561 if (CHECK_FAIL(prog_fd < 0))
562 goto out;
563
564 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
565 if (CHECK_FAIL(err))
566 goto out;
567 }
568
569 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
570 err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);
571 if (CHECK_FAIL(err))
572 goto out;
573
574 err = bpf_prog_test_run_opts(main_fd, &topts);
575 ASSERT_OK(err, "tailcall");
576 ASSERT_EQ(topts.retval, i, "tailcall retval");
577 }
578
579 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
580 err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);
581 if (CHECK_FAIL(err))
582 goto out;
583
584 err = bpf_map_delete_elem(map_fd, &i);
585 if (CHECK_FAIL(err))
586 goto out;
587
588 err = bpf_prog_test_run_opts(main_fd, &topts);
589 ASSERT_OK(err, "tailcall");
590 ASSERT_EQ(topts.retval, 3, "tailcall retval");
591 }
592 out:
593 bpf_object__close(obj);
594 }
595
596 /* test_tailcall_bpf2bpf_1 purpose is to make sure that tailcalls are working
597 * correctly in correlation with BPF subprograms
598 */
test_tailcall_bpf2bpf_1(void)599 static void test_tailcall_bpf2bpf_1(void)
600 {
601 int err, map_fd, prog_fd, main_fd, i;
602 struct bpf_map *prog_array;
603 struct bpf_program *prog;
604 struct bpf_object *obj;
605 char prog_name[32];
606 LIBBPF_OPTS(bpf_test_run_opts, topts,
607 .data_in = &pkt_v4,
608 .data_size_in = sizeof(pkt_v4),
609 .repeat = 1,
610 );
611
612 err = bpf_prog_test_load("tailcall_bpf2bpf1.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
613 &obj, &prog_fd);
614 if (CHECK_FAIL(err))
615 return;
616
617 prog = bpf_object__find_program_by_name(obj, "entry");
618 if (CHECK_FAIL(!prog))
619 goto out;
620
621 main_fd = bpf_program__fd(prog);
622 if (CHECK_FAIL(main_fd < 0))
623 goto out;
624
625 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
626 if (CHECK_FAIL(!prog_array))
627 goto out;
628
629 map_fd = bpf_map__fd(prog_array);
630 if (CHECK_FAIL(map_fd < 0))
631 goto out;
632
633 /* nop -> jmp */
634 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
635 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
636
637 prog = bpf_object__find_program_by_name(obj, prog_name);
638 if (CHECK_FAIL(!prog))
639 goto out;
640
641 prog_fd = bpf_program__fd(prog);
642 if (CHECK_FAIL(prog_fd < 0))
643 goto out;
644
645 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
646 if (CHECK_FAIL(err))
647 goto out;
648 }
649
650 err = bpf_prog_test_run_opts(main_fd, &topts);
651 ASSERT_OK(err, "tailcall");
652 ASSERT_EQ(topts.retval, 1, "tailcall retval");
653
654 /* jmp -> nop, call subprog that will do tailcall */
655 i = 1;
656 err = bpf_map_delete_elem(map_fd, &i);
657 if (CHECK_FAIL(err))
658 goto out;
659
660 err = bpf_prog_test_run_opts(main_fd, &topts);
661 ASSERT_OK(err, "tailcall");
662 ASSERT_OK(topts.retval, "tailcall retval");
663
664 /* make sure that subprog can access ctx and entry prog that
665 * called this subprog can properly return
666 */
667 i = 0;
668 err = bpf_map_delete_elem(map_fd, &i);
669 if (CHECK_FAIL(err))
670 goto out;
671
672 err = bpf_prog_test_run_opts(main_fd, &topts);
673 ASSERT_OK(err, "tailcall");
674 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 2, "tailcall retval");
675 out:
676 bpf_object__close(obj);
677 }
678
679 /* test_tailcall_bpf2bpf_2 checks that the count value of the tail call limit
680 * enforcement matches with expectations when tailcall is preceded with
681 * bpf2bpf call.
682 */
test_tailcall_bpf2bpf_2(void)683 static void test_tailcall_bpf2bpf_2(void)
684 {
685 int err, map_fd, prog_fd, main_fd, data_fd, i, val;
686 struct bpf_map *prog_array, *data_map;
687 struct bpf_program *prog;
688 struct bpf_object *obj;
689 char buff[128] = {};
690 LIBBPF_OPTS(bpf_test_run_opts, topts,
691 .data_in = buff,
692 .data_size_in = sizeof(buff),
693 .repeat = 1,
694 );
695
696 err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
697 &obj, &prog_fd);
698 if (CHECK_FAIL(err))
699 return;
700
701 prog = bpf_object__find_program_by_name(obj, "entry");
702 if (CHECK_FAIL(!prog))
703 goto out;
704
705 main_fd = bpf_program__fd(prog);
706 if (CHECK_FAIL(main_fd < 0))
707 goto out;
708
709 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
710 if (CHECK_FAIL(!prog_array))
711 goto out;
712
713 map_fd = bpf_map__fd(prog_array);
714 if (CHECK_FAIL(map_fd < 0))
715 goto out;
716
717 prog = bpf_object__find_program_by_name(obj, "classifier_0");
718 if (CHECK_FAIL(!prog))
719 goto out;
720
721 prog_fd = bpf_program__fd(prog);
722 if (CHECK_FAIL(prog_fd < 0))
723 goto out;
724
725 i = 0;
726 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
727 if (CHECK_FAIL(err))
728 goto out;
729
730 err = bpf_prog_test_run_opts(main_fd, &topts);
731 ASSERT_OK(err, "tailcall");
732 ASSERT_EQ(topts.retval, 1, "tailcall retval");
733
734 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
735 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
736 goto out;
737
738 data_fd = bpf_map__fd(data_map);
739 if (CHECK_FAIL(data_fd < 0))
740 goto out;
741
742 i = 0;
743 err = bpf_map_lookup_elem(data_fd, &i, &val);
744 ASSERT_OK(err, "tailcall count");
745 ASSERT_EQ(val, 33, "tailcall count");
746
747 i = 0;
748 err = bpf_map_delete_elem(map_fd, &i);
749 if (CHECK_FAIL(err))
750 goto out;
751
752 err = bpf_prog_test_run_opts(main_fd, &topts);
753 ASSERT_OK(err, "tailcall");
754 ASSERT_OK(topts.retval, "tailcall retval");
755 out:
756 bpf_object__close(obj);
757 }
758
759 /* test_tailcall_bpf2bpf_3 checks that non-trivial amount of stack (up to
760 * 256 bytes) can be used within bpf subprograms that have the tailcalls
761 * in them
762 */
test_tailcall_bpf2bpf_3(void)763 static void test_tailcall_bpf2bpf_3(void)
764 {
765 int err, map_fd, prog_fd, main_fd, i;
766 struct bpf_map *prog_array;
767 struct bpf_program *prog;
768 struct bpf_object *obj;
769 char prog_name[32];
770 LIBBPF_OPTS(bpf_test_run_opts, topts,
771 .data_in = &pkt_v4,
772 .data_size_in = sizeof(pkt_v4),
773 .repeat = 1,
774 );
775
776 err = bpf_prog_test_load("tailcall_bpf2bpf3.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
777 &obj, &prog_fd);
778 if (CHECK_FAIL(err))
779 return;
780
781 prog = bpf_object__find_program_by_name(obj, "entry");
782 if (CHECK_FAIL(!prog))
783 goto out;
784
785 main_fd = bpf_program__fd(prog);
786 if (CHECK_FAIL(main_fd < 0))
787 goto out;
788
789 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
790 if (CHECK_FAIL(!prog_array))
791 goto out;
792
793 map_fd = bpf_map__fd(prog_array);
794 if (CHECK_FAIL(map_fd < 0))
795 goto out;
796
797 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
798 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
799
800 prog = bpf_object__find_program_by_name(obj, prog_name);
801 if (CHECK_FAIL(!prog))
802 goto out;
803
804 prog_fd = bpf_program__fd(prog);
805 if (CHECK_FAIL(prog_fd < 0))
806 goto out;
807
808 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
809 if (CHECK_FAIL(err))
810 goto out;
811 }
812
813 err = bpf_prog_test_run_opts(main_fd, &topts);
814 ASSERT_OK(err, "tailcall");
815 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 3, "tailcall retval");
816
817 i = 1;
818 err = bpf_map_delete_elem(map_fd, &i);
819 if (CHECK_FAIL(err))
820 goto out;
821
822 err = bpf_prog_test_run_opts(main_fd, &topts);
823 ASSERT_OK(err, "tailcall");
824 ASSERT_EQ(topts.retval, sizeof(pkt_v4), "tailcall retval");
825
826 i = 0;
827 err = bpf_map_delete_elem(map_fd, &i);
828 if (CHECK_FAIL(err))
829 goto out;
830
831 err = bpf_prog_test_run_opts(main_fd, &topts);
832 ASSERT_OK(err, "tailcall");
833 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 2, "tailcall retval");
834 out:
835 bpf_object__close(obj);
836 }
837
838 #include "tailcall_bpf2bpf4.skel.h"
839
840 /* test_tailcall_bpf2bpf_4 checks that tailcall counter is correctly preserved
841 * across tailcalls combined with bpf2bpf calls. for making sure that tailcall
842 * counter behaves correctly, bpf program will go through following flow:
843 *
844 * entry -> entry_subprog -> tailcall0 -> bpf_func0 -> subprog0 ->
845 * -> tailcall1 -> bpf_func1 -> subprog1 -> tailcall2 -> bpf_func2 ->
846 * subprog2 [here bump global counter] --------^
847 *
848 * We go through first two tailcalls and start counting from the subprog2 where
849 * the loop begins. At the end of the test make sure that the global counter is
850 * equal to 31, because tailcall counter includes the first two tailcalls
851 * whereas global counter is incremented only on loop presented on flow above.
852 *
853 * The noise parameter is used to insert bpf_map_update calls into the logic
854 * to force verifier to patch instructions. This allows us to ensure jump
855 * logic remains correct with instruction movement.
856 */
test_tailcall_bpf2bpf_4(bool noise)857 static void test_tailcall_bpf2bpf_4(bool noise)
858 {
859 int err, map_fd, prog_fd, main_fd, data_fd, i;
860 struct tailcall_bpf2bpf4__bss val;
861 struct bpf_map *prog_array, *data_map;
862 struct bpf_program *prog;
863 struct bpf_object *obj;
864 char prog_name[32];
865 LIBBPF_OPTS(bpf_test_run_opts, topts,
866 .data_in = &pkt_v4,
867 .data_size_in = sizeof(pkt_v4),
868 .repeat = 1,
869 );
870
871 err = bpf_prog_test_load("tailcall_bpf2bpf4.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
872 &obj, &prog_fd);
873 if (CHECK_FAIL(err))
874 return;
875
876 prog = bpf_object__find_program_by_name(obj, "entry");
877 if (CHECK_FAIL(!prog))
878 goto out;
879
880 main_fd = bpf_program__fd(prog);
881 if (CHECK_FAIL(main_fd < 0))
882 goto out;
883
884 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
885 if (CHECK_FAIL(!prog_array))
886 goto out;
887
888 map_fd = bpf_map__fd(prog_array);
889 if (CHECK_FAIL(map_fd < 0))
890 goto out;
891
892 for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
893 snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
894
895 prog = bpf_object__find_program_by_name(obj, prog_name);
896 if (CHECK_FAIL(!prog))
897 goto out;
898
899 prog_fd = bpf_program__fd(prog);
900 if (CHECK_FAIL(prog_fd < 0))
901 goto out;
902
903 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
904 if (CHECK_FAIL(err))
905 goto out;
906 }
907
908 data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
909 if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
910 goto out;
911
912 data_fd = bpf_map__fd(data_map);
913 if (CHECK_FAIL(data_fd < 0))
914 goto out;
915
916 i = 0;
917 val.noise = noise;
918 val.count = 0;
919 err = bpf_map_update_elem(data_fd, &i, &val, BPF_ANY);
920 if (CHECK_FAIL(err))
921 goto out;
922
923 err = bpf_prog_test_run_opts(main_fd, &topts);
924 ASSERT_OK(err, "tailcall");
925 ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 3, "tailcall retval");
926
927 i = 0;
928 err = bpf_map_lookup_elem(data_fd, &i, &val);
929 ASSERT_OK(err, "tailcall count");
930 ASSERT_EQ(val.count, 31, "tailcall count");
931
932 out:
933 bpf_object__close(obj);
934 }
935
936 #include "tailcall_bpf2bpf6.skel.h"
937
938 /* Tail call counting works even when there is data on stack which is
939 * not aligned to 8 bytes.
940 */
test_tailcall_bpf2bpf_6(void)941 static void test_tailcall_bpf2bpf_6(void)
942 {
943 struct tailcall_bpf2bpf6 *obj;
944 int err, map_fd, prog_fd, main_fd, data_fd, i, val;
945 LIBBPF_OPTS(bpf_test_run_opts, topts,
946 .data_in = &pkt_v4,
947 .data_size_in = sizeof(pkt_v4),
948 .repeat = 1,
949 );
950
951 obj = tailcall_bpf2bpf6__open_and_load();
952 if (!ASSERT_OK_PTR(obj, "open and load"))
953 return;
954
955 main_fd = bpf_program__fd(obj->progs.entry);
956 if (!ASSERT_GE(main_fd, 0, "entry prog fd"))
957 goto out;
958
959 map_fd = bpf_map__fd(obj->maps.jmp_table);
960 if (!ASSERT_GE(map_fd, 0, "jmp_table map fd"))
961 goto out;
962
963 prog_fd = bpf_program__fd(obj->progs.classifier_0);
964 if (!ASSERT_GE(prog_fd, 0, "classifier_0 prog fd"))
965 goto out;
966
967 i = 0;
968 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
969 if (!ASSERT_OK(err, "jmp_table map update"))
970 goto out;
971
972 err = bpf_prog_test_run_opts(main_fd, &topts);
973 ASSERT_OK(err, "entry prog test run");
974 ASSERT_EQ(topts.retval, 0, "tailcall retval");
975
976 data_fd = bpf_map__fd(obj->maps.bss);
977 if (!ASSERT_GE(data_fd, 0, "bss map fd"))
978 goto out;
979
980 i = 0;
981 err = bpf_map_lookup_elem(data_fd, &i, &val);
982 ASSERT_OK(err, "bss map lookup");
983 ASSERT_EQ(val, 1, "done flag is set");
984
985 out:
986 tailcall_bpf2bpf6__destroy(obj);
987 }
988
989 /* test_tailcall_bpf2bpf_fentry checks that the count value of the tail call
990 * limit enforcement matches with expectations when tailcall is preceded with
991 * bpf2bpf call, and the bpf2bpf call is traced by fentry.
992 */
test_tailcall_bpf2bpf_fentry(void)993 static void test_tailcall_bpf2bpf_fentry(void)
994 {
995 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", true, false);
996 }
997
998 /* test_tailcall_bpf2bpf_fexit checks that the count value of the tail call
999 * limit enforcement matches with expectations when tailcall is preceded with
1000 * bpf2bpf call, and the bpf2bpf call is traced by fexit.
1001 */
test_tailcall_bpf2bpf_fexit(void)1002 static void test_tailcall_bpf2bpf_fexit(void)
1003 {
1004 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", false, true);
1005 }
1006
1007 /* test_tailcall_bpf2bpf_fentry_fexit checks that the count value of the tail
1008 * call limit enforcement matches with expectations when tailcall is preceded
1009 * with bpf2bpf call, and the bpf2bpf call is traced by both fentry and fexit.
1010 */
test_tailcall_bpf2bpf_fentry_fexit(void)1011 static void test_tailcall_bpf2bpf_fentry_fexit(void)
1012 {
1013 test_tailcall_count("tailcall_bpf2bpf2.bpf.o", true, true);
1014 }
1015
1016 /* test_tailcall_bpf2bpf_fentry_entry checks that the count value of the tail
1017 * call limit enforcement matches with expectations when tailcall is preceded
1018 * with bpf2bpf call, and the bpf2bpf caller is traced by fentry.
1019 */
test_tailcall_bpf2bpf_fentry_entry(void)1020 static void test_tailcall_bpf2bpf_fentry_entry(void)
1021 {
1022 struct bpf_object *tgt_obj = NULL, *fentry_obj = NULL;
1023 int err, map_fd, prog_fd, data_fd, i, val;
1024 struct bpf_map *prog_array, *data_map;
1025 struct bpf_link *fentry_link = NULL;
1026 struct bpf_program *prog;
1027 char buff[128] = {};
1028
1029 LIBBPF_OPTS(bpf_test_run_opts, topts,
1030 .data_in = buff,
1031 .data_size_in = sizeof(buff),
1032 .repeat = 1,
1033 );
1034
1035 err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o",
1036 BPF_PROG_TYPE_SCHED_CLS,
1037 &tgt_obj, &prog_fd);
1038 if (!ASSERT_OK(err, "load tgt_obj"))
1039 return;
1040
1041 prog_array = bpf_object__find_map_by_name(tgt_obj, "jmp_table");
1042 if (!ASSERT_OK_PTR(prog_array, "find jmp_table map"))
1043 goto out;
1044
1045 map_fd = bpf_map__fd(prog_array);
1046 if (!ASSERT_FALSE(map_fd < 0, "find jmp_table map fd"))
1047 goto out;
1048
1049 prog = bpf_object__find_program_by_name(tgt_obj, "classifier_0");
1050 if (!ASSERT_OK_PTR(prog, "find classifier_0 prog"))
1051 goto out;
1052
1053 prog_fd = bpf_program__fd(prog);
1054 if (!ASSERT_FALSE(prog_fd < 0, "find classifier_0 prog fd"))
1055 goto out;
1056
1057 i = 0;
1058 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
1059 if (!ASSERT_OK(err, "update jmp_table"))
1060 goto out;
1061
1062 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o",
1063 NULL);
1064 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file"))
1065 goto out;
1066
1067 prog = bpf_object__find_program_by_name(fentry_obj, "fentry");
1068 if (!ASSERT_OK_PTR(prog, "find fentry prog"))
1069 goto out;
1070
1071 err = bpf_program__set_attach_target(prog, prog_fd, "classifier_0");
1072 if (!ASSERT_OK(err, "set_attach_target classifier_0"))
1073 goto out;
1074
1075 err = bpf_object__load(fentry_obj);
1076 if (!ASSERT_OK(err, "load fentry_obj"))
1077 goto out;
1078
1079 fentry_link = bpf_program__attach_trace(prog);
1080 if (!ASSERT_OK_PTR(fentry_link, "attach_trace"))
1081 goto out;
1082
1083 err = bpf_prog_test_run_opts(prog_fd, &topts);
1084 ASSERT_OK(err, "tailcall");
1085 ASSERT_EQ(topts.retval, 1, "tailcall retval");
1086
1087 data_map = bpf_object__find_map_by_name(tgt_obj, "tailcall.bss");
1088 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),
1089 "find tailcall.bss map"))
1090 goto out;
1091
1092 data_fd = bpf_map__fd(data_map);
1093 if (!ASSERT_FALSE(data_fd < 0, "find tailcall.bss map fd"))
1094 goto out;
1095
1096 i = 0;
1097 err = bpf_map_lookup_elem(data_fd, &i, &val);
1098 ASSERT_OK(err, "tailcall count");
1099 ASSERT_EQ(val, 34, "tailcall count");
1100
1101 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss");
1102 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),
1103 "find tailcall_bpf2bpf_fentry.bss map"))
1104 goto out;
1105
1106 data_fd = bpf_map__fd(data_map);
1107 if (!ASSERT_FALSE(data_fd < 0,
1108 "find tailcall_bpf2bpf_fentry.bss map fd"))
1109 goto out;
1110
1111 i = 0;
1112 err = bpf_map_lookup_elem(data_fd, &i, &val);
1113 ASSERT_OK(err, "fentry count");
1114 ASSERT_EQ(val, 1, "fentry count");
1115
1116 out:
1117 bpf_link__destroy(fentry_link);
1118 bpf_object__close(fentry_obj);
1119 bpf_object__close(tgt_obj);
1120 }
1121
1122 #define JMP_TABLE "/sys/fs/bpf/jmp_table"
1123
1124 static int poke_thread_exit;
1125
poke_update(void * arg)1126 static void *poke_update(void *arg)
1127 {
1128 __u32 zero = 0, prog1_fd, prog2_fd, map_fd;
1129 struct tailcall_poke *call = arg;
1130
1131 map_fd = bpf_map__fd(call->maps.jmp_table);
1132 prog1_fd = bpf_program__fd(call->progs.call1);
1133 prog2_fd = bpf_program__fd(call->progs.call2);
1134
1135 while (!poke_thread_exit) {
1136 bpf_map_update_elem(map_fd, &zero, &prog1_fd, BPF_ANY);
1137 bpf_map_update_elem(map_fd, &zero, &prog2_fd, BPF_ANY);
1138 }
1139
1140 return NULL;
1141 }
1142
1143 /*
1144 * We are trying to hit prog array update during another program load
1145 * that shares the same prog array map.
1146 *
1147 * For that we share the jmp_table map between two skeleton instances
1148 * by pinning the jmp_table to same path. Then first skeleton instance
1149 * periodically updates jmp_table in 'poke update' thread while we load
1150 * the second skeleton instance in the main thread.
1151 */
test_tailcall_poke(void)1152 static void test_tailcall_poke(void)
1153 {
1154 struct tailcall_poke *call, *test;
1155 int err, cnt = 10;
1156 pthread_t thread;
1157
1158 unlink(JMP_TABLE);
1159
1160 call = tailcall_poke__open_and_load();
1161 if (!ASSERT_OK_PTR(call, "tailcall_poke__open"))
1162 return;
1163
1164 err = bpf_map__pin(call->maps.jmp_table, JMP_TABLE);
1165 if (!ASSERT_OK(err, "bpf_map__pin"))
1166 goto out;
1167
1168 err = pthread_create(&thread, NULL, poke_update, call);
1169 if (!ASSERT_OK(err, "new toggler"))
1170 goto out;
1171
1172 while (cnt--) {
1173 test = tailcall_poke__open();
1174 if (!ASSERT_OK_PTR(test, "tailcall_poke__open"))
1175 break;
1176
1177 err = bpf_map__set_pin_path(test->maps.jmp_table, JMP_TABLE);
1178 if (!ASSERT_OK(err, "bpf_map__pin")) {
1179 tailcall_poke__destroy(test);
1180 break;
1181 }
1182
1183 bpf_program__set_autoload(test->progs.test, true);
1184 bpf_program__set_autoload(test->progs.call1, false);
1185 bpf_program__set_autoload(test->progs.call2, false);
1186
1187 err = tailcall_poke__load(test);
1188 tailcall_poke__destroy(test);
1189 if (!ASSERT_OK(err, "tailcall_poke__load"))
1190 break;
1191 }
1192
1193 poke_thread_exit = 1;
1194 ASSERT_OK(pthread_join(thread, NULL), "pthread_join");
1195
1196 out:
1197 bpf_map__unpin(call->maps.jmp_table, JMP_TABLE);
1198 tailcall_poke__destroy(call);
1199 }
1200
test_tailcall_hierarchy_count(const char * which,bool test_fentry,bool test_fexit,bool test_fentry_entry)1201 static void test_tailcall_hierarchy_count(const char *which, bool test_fentry,
1202 bool test_fexit,
1203 bool test_fentry_entry)
1204 {
1205 int err, map_fd, prog_fd, main_data_fd, fentry_data_fd = 0, fexit_data_fd = 0, i, val;
1206 struct bpf_object *obj = NULL, *fentry_obj = NULL, *fexit_obj = NULL;
1207 struct bpf_link *fentry_link = NULL, *fexit_link = NULL;
1208 struct bpf_program *prog, *fentry_prog;
1209 struct bpf_map *prog_array, *data_map;
1210 int fentry_prog_fd;
1211 char buff[128] = {};
1212
1213 LIBBPF_OPTS(bpf_test_run_opts, topts,
1214 .data_in = buff,
1215 .data_size_in = sizeof(buff),
1216 .repeat = 1,
1217 );
1218
1219 err = bpf_prog_test_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj,
1220 &prog_fd);
1221 if (!ASSERT_OK(err, "load obj"))
1222 return;
1223
1224 prog = bpf_object__find_program_by_name(obj, "entry");
1225 if (!ASSERT_OK_PTR(prog, "find entry prog"))
1226 goto out;
1227
1228 prog_fd = bpf_program__fd(prog);
1229 if (!ASSERT_GE(prog_fd, 0, "prog_fd"))
1230 goto out;
1231
1232 if (test_fentry_entry) {
1233 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_hierarchy_fentry.bpf.o",
1234 NULL);
1235 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file"))
1236 goto out;
1237
1238 fentry_prog = bpf_object__find_program_by_name(fentry_obj,
1239 "fentry");
1240 if (!ASSERT_OK_PTR(prog, "find fentry prog"))
1241 goto out;
1242
1243 err = bpf_program__set_attach_target(fentry_prog, prog_fd,
1244 "entry");
1245 if (!ASSERT_OK(err, "set_attach_target entry"))
1246 goto out;
1247
1248 err = bpf_object__load(fentry_obj);
1249 if (!ASSERT_OK(err, "load fentry_obj"))
1250 goto out;
1251
1252 fentry_link = bpf_program__attach_trace(fentry_prog);
1253 if (!ASSERT_OK_PTR(fentry_link, "attach_trace"))
1254 goto out;
1255
1256 fentry_prog_fd = bpf_program__fd(fentry_prog);
1257 if (!ASSERT_GE(fentry_prog_fd, 0, "fentry_prog_fd"))
1258 goto out;
1259
1260 prog_array = bpf_object__find_map_by_name(fentry_obj, "jmp_table");
1261 if (!ASSERT_OK_PTR(prog_array, "find jmp_table"))
1262 goto out;
1263
1264 map_fd = bpf_map__fd(prog_array);
1265 if (!ASSERT_GE(map_fd, 0, "map_fd"))
1266 goto out;
1267
1268 i = 0;
1269 err = bpf_map_update_elem(map_fd, &i, &fentry_prog_fd, BPF_ANY);
1270 if (!ASSERT_OK(err, "update jmp_table"))
1271 goto out;
1272
1273 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss");
1274 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),
1275 "find data_map"))
1276 goto out;
1277
1278 } else {
1279 prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
1280 if (!ASSERT_OK_PTR(prog_array, "find jmp_table"))
1281 goto out;
1282
1283 map_fd = bpf_map__fd(prog_array);
1284 if (!ASSERT_GE(map_fd, 0, "map_fd"))
1285 goto out;
1286
1287 i = 0;
1288 err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
1289 if (!ASSERT_OK(err, "update jmp_table"))
1290 goto out;
1291
1292 data_map = bpf_object__find_map_by_name(obj, ".bss");
1293 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),
1294 "find data_map"))
1295 goto out;
1296 }
1297
1298 if (test_fentry) {
1299 fentry_obj = bpf_object__open_file("tailcall_bpf2bpf_fentry.bpf.o",
1300 NULL);
1301 if (!ASSERT_OK_PTR(fentry_obj, "open fentry_obj file"))
1302 goto out;
1303
1304 prog = bpf_object__find_program_by_name(fentry_obj, "fentry");
1305 if (!ASSERT_OK_PTR(prog, "find fentry prog"))
1306 goto out;
1307
1308 err = bpf_program__set_attach_target(prog, prog_fd,
1309 "subprog_tail");
1310 if (!ASSERT_OK(err, "set_attach_target subprog_tail"))
1311 goto out;
1312
1313 err = bpf_object__load(fentry_obj);
1314 if (!ASSERT_OK(err, "load fentry_obj"))
1315 goto out;
1316
1317 fentry_link = bpf_program__attach_trace(prog);
1318 if (!ASSERT_OK_PTR(fentry_link, "attach_trace"))
1319 goto out;
1320 }
1321
1322 if (test_fexit) {
1323 fexit_obj = bpf_object__open_file("tailcall_bpf2bpf_fexit.bpf.o",
1324 NULL);
1325 if (!ASSERT_OK_PTR(fexit_obj, "open fexit_obj file"))
1326 goto out;
1327
1328 prog = bpf_object__find_program_by_name(fexit_obj, "fexit");
1329 if (!ASSERT_OK_PTR(prog, "find fexit prog"))
1330 goto out;
1331
1332 err = bpf_program__set_attach_target(prog, prog_fd,
1333 "subprog_tail");
1334 if (!ASSERT_OK(err, "set_attach_target subprog_tail"))
1335 goto out;
1336
1337 err = bpf_object__load(fexit_obj);
1338 if (!ASSERT_OK(err, "load fexit_obj"))
1339 goto out;
1340
1341 fexit_link = bpf_program__attach_trace(prog);
1342 if (!ASSERT_OK_PTR(fexit_link, "attach_trace"))
1343 goto out;
1344 }
1345
1346 err = bpf_prog_test_run_opts(prog_fd, &topts);
1347 ASSERT_OK(err, "tailcall");
1348 ASSERT_EQ(topts.retval, 1, "tailcall retval");
1349
1350 main_data_fd = bpf_map__fd(data_map);
1351 if (!ASSERT_GE(main_data_fd, 0, "main_data_fd"))
1352 goto out;
1353
1354 i = 0;
1355 err = bpf_map_lookup_elem(main_data_fd, &i, &val);
1356 ASSERT_OK(err, "tailcall count");
1357 ASSERT_EQ(val, 34, "tailcall count");
1358
1359 if (test_fentry) {
1360 data_map = bpf_object__find_map_by_name(fentry_obj, ".bss");
1361 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),
1362 "find tailcall_bpf2bpf_fentry.bss map"))
1363 goto out;
1364
1365 fentry_data_fd = bpf_map__fd(data_map);
1366 if (!ASSERT_GE(fentry_data_fd, 0,
1367 "find tailcall_bpf2bpf_fentry.bss map fd"))
1368 goto out;
1369
1370 i = 0;
1371 err = bpf_map_lookup_elem(fentry_data_fd, &i, &val);
1372 ASSERT_OK(err, "fentry count");
1373 ASSERT_EQ(val, 68, "fentry count");
1374 }
1375
1376 if (test_fexit) {
1377 data_map = bpf_object__find_map_by_name(fexit_obj, ".bss");
1378 if (!ASSERT_FALSE(!data_map || !bpf_map__is_internal(data_map),
1379 "find tailcall_bpf2bpf_fexit.bss map"))
1380 goto out;
1381
1382 fexit_data_fd = bpf_map__fd(data_map);
1383 if (!ASSERT_GE(fexit_data_fd, 0,
1384 "find tailcall_bpf2bpf_fexit.bss map fd"))
1385 goto out;
1386
1387 i = 0;
1388 err = bpf_map_lookup_elem(fexit_data_fd, &i, &val);
1389 ASSERT_OK(err, "fexit count");
1390 ASSERT_EQ(val, 68, "fexit count");
1391 }
1392
1393 i = 0;
1394 err = bpf_map_delete_elem(map_fd, &i);
1395 if (!ASSERT_OK(err, "delete_elem from jmp_table"))
1396 goto out;
1397
1398 err = bpf_prog_test_run_opts(prog_fd, &topts);
1399 ASSERT_OK(err, "tailcall");
1400 ASSERT_EQ(topts.retval, 1, "tailcall retval");
1401
1402 i = 0;
1403 err = bpf_map_lookup_elem(main_data_fd, &i, &val);
1404 ASSERT_OK(err, "tailcall count");
1405 ASSERT_EQ(val, 35, "tailcall count");
1406
1407 if (test_fentry) {
1408 i = 0;
1409 err = bpf_map_lookup_elem(fentry_data_fd, &i, &val);
1410 ASSERT_OK(err, "fentry count");
1411 ASSERT_EQ(val, 70, "fentry count");
1412 }
1413
1414 if (test_fexit) {
1415 i = 0;
1416 err = bpf_map_lookup_elem(fexit_data_fd, &i, &val);
1417 ASSERT_OK(err, "fexit count");
1418 ASSERT_EQ(val, 70, "fexit count");
1419 }
1420
1421 out:
1422 bpf_link__destroy(fentry_link);
1423 bpf_link__destroy(fexit_link);
1424 bpf_object__close(fentry_obj);
1425 bpf_object__close(fexit_obj);
1426 bpf_object__close(obj);
1427 }
1428
1429 /* test_tailcall_bpf2bpf_hierarchy_1 checks that the count value of the tail
1430 * call limit enforcement matches with expectations when tailcalls are preceded
1431 * with two bpf2bpf calls.
1432 *
1433 * subprog --tailcall-> entry
1434 * entry <
1435 * subprog --tailcall-> entry
1436 */
test_tailcall_bpf2bpf_hierarchy_1(void)1437 static void test_tailcall_bpf2bpf_hierarchy_1(void)
1438 {
1439 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o",
1440 false, false, false);
1441 }
1442
1443 /* test_tailcall_bpf2bpf_hierarchy_fentry checks that the count value of the
1444 * tail call limit enforcement matches with expectations when tailcalls are
1445 * preceded with two bpf2bpf calls, and the two subprogs are traced by fentry.
1446 */
test_tailcall_bpf2bpf_hierarchy_fentry(void)1447 static void test_tailcall_bpf2bpf_hierarchy_fentry(void)
1448 {
1449 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o",
1450 true, false, false);
1451 }
1452
1453 /* test_tailcall_bpf2bpf_hierarchy_fexit checks that the count value of the tail
1454 * call limit enforcement matches with expectations when tailcalls are preceded
1455 * with two bpf2bpf calls, and the two subprogs are traced by fexit.
1456 */
test_tailcall_bpf2bpf_hierarchy_fexit(void)1457 static void test_tailcall_bpf2bpf_hierarchy_fexit(void)
1458 {
1459 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o",
1460 false, true, false);
1461 }
1462
1463 /* test_tailcall_bpf2bpf_hierarchy_fentry_fexit checks that the count value of
1464 * the tail call limit enforcement matches with expectations when tailcalls are
1465 * preceded with two bpf2bpf calls, and the two subprogs are traced by both
1466 * fentry and fexit.
1467 */
test_tailcall_bpf2bpf_hierarchy_fentry_fexit(void)1468 static void test_tailcall_bpf2bpf_hierarchy_fentry_fexit(void)
1469 {
1470 test_tailcall_hierarchy_count("tailcall_bpf2bpf_hierarchy1.bpf.o",
1471 true, true, false);
1472 }
1473
1474 /* test_tailcall_bpf2bpf_hierarchy_fentry_entry checks that the count value of
1475 * the tail call limit enforcement matches with expectations when tailcalls are
1476 * preceded with two bpf2bpf calls in fentry.
1477 */
test_tailcall_bpf2bpf_hierarchy_fentry_entry(void)1478 static void test_tailcall_bpf2bpf_hierarchy_fentry_entry(void)
1479 {
1480 test_tailcall_hierarchy_count("tc_dummy.bpf.o", false, false, true);
1481 }
1482
1483 /* test_tailcall_bpf2bpf_hierarchy_2 checks that the count value of the tail
1484 * call limit enforcement matches with expectations:
1485 *
1486 * subprog_tail0 --tailcall-> classifier_0 -> subprog_tail0
1487 * entry <
1488 * subprog_tail1 --tailcall-> classifier_1 -> subprog_tail1
1489 */
test_tailcall_bpf2bpf_hierarchy_2(void)1490 static void test_tailcall_bpf2bpf_hierarchy_2(void)
1491 {
1492 RUN_TESTS(tailcall_bpf2bpf_hierarchy2);
1493 }
1494
1495 /* test_tailcall_bpf2bpf_hierarchy_3 checks that the count value of the tail
1496 * call limit enforcement matches with expectations:
1497 *
1498 * subprog with jmp_table0 to classifier_0
1499 * entry --tailcall-> classifier_0 <
1500 * subprog with jmp_table1 to classifier_0
1501 */
test_tailcall_bpf2bpf_hierarchy_3(void)1502 static void test_tailcall_bpf2bpf_hierarchy_3(void)
1503 {
1504 RUN_TESTS(tailcall_bpf2bpf_hierarchy3);
1505 }
1506
1507 /* test_tailcall_freplace checks that the freplace prog fails to update the
1508 * prog_array map, no matter whether the freplace prog attaches to its target.
1509 */
test_tailcall_freplace(void)1510 static void test_tailcall_freplace(void)
1511 {
1512 struct tailcall_freplace *freplace_skel = NULL;
1513 struct bpf_link *freplace_link = NULL;
1514 struct bpf_program *freplace_prog;
1515 struct tc_bpf2bpf *tc_skel = NULL;
1516 int prog_fd, tc_prog_fd, map_fd;
1517 char buff[128] = {};
1518 int err, key;
1519
1520 LIBBPF_OPTS(bpf_test_run_opts, topts,
1521 .data_in = buff,
1522 .data_size_in = sizeof(buff),
1523 .repeat = 1,
1524 );
1525
1526 freplace_skel = tailcall_freplace__open();
1527 if (!ASSERT_OK_PTR(freplace_skel, "tailcall_freplace__open"))
1528 return;
1529
1530 tc_skel = tc_bpf2bpf__open_and_load();
1531 if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load"))
1532 goto out;
1533
1534 tc_prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);
1535 freplace_prog = freplace_skel->progs.entry_freplace;
1536 err = bpf_program__set_attach_target(freplace_prog, tc_prog_fd,
1537 "subprog_tc");
1538 if (!ASSERT_OK(err, "set_attach_target"))
1539 goto out;
1540
1541 err = tailcall_freplace__load(freplace_skel);
1542 if (!ASSERT_OK(err, "tailcall_freplace__load"))
1543 goto out;
1544
1545 map_fd = bpf_map__fd(freplace_skel->maps.jmp_table);
1546 prog_fd = bpf_program__fd(freplace_prog);
1547 key = 0;
1548 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1549 ASSERT_ERR(err, "update jmp_table failure");
1550
1551 freplace_link = bpf_program__attach_freplace(freplace_prog, tc_prog_fd,
1552 "subprog_tc");
1553 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace"))
1554 goto out;
1555
1556 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1557 ASSERT_ERR(err, "update jmp_table failure");
1558
1559 out:
1560 bpf_link__destroy(freplace_link);
1561 tailcall_freplace__destroy(freplace_skel);
1562 tc_bpf2bpf__destroy(tc_skel);
1563 }
1564
1565 /* test_tailcall_bpf2bpf_freplace checks the failure that fails to attach a tail
1566 * callee prog with freplace prog or fails to update an extended prog to
1567 * prog_array map.
1568 */
test_tailcall_bpf2bpf_freplace(void)1569 static void test_tailcall_bpf2bpf_freplace(void)
1570 {
1571 struct tailcall_freplace *freplace_skel = NULL;
1572 struct bpf_link *freplace_link = NULL;
1573 struct tc_bpf2bpf *tc_skel = NULL;
1574 char buff[128] = {};
1575 int prog_fd, map_fd;
1576 int err, key;
1577
1578 LIBBPF_OPTS(bpf_test_run_opts, topts,
1579 .data_in = buff,
1580 .data_size_in = sizeof(buff),
1581 .repeat = 1,
1582 );
1583
1584 tc_skel = tc_bpf2bpf__open_and_load();
1585 if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load"))
1586 goto out;
1587
1588 prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);
1589 freplace_skel = tailcall_freplace__open();
1590 if (!ASSERT_OK_PTR(freplace_skel, "tailcall_freplace__open"))
1591 goto out;
1592
1593 err = bpf_program__set_attach_target(freplace_skel->progs.entry_freplace,
1594 prog_fd, "subprog_tc");
1595 if (!ASSERT_OK(err, "set_attach_target"))
1596 goto out;
1597
1598 err = tailcall_freplace__load(freplace_skel);
1599 if (!ASSERT_OK(err, "tailcall_freplace__load"))
1600 goto out;
1601
1602 /* OK to attach then detach freplace prog. */
1603
1604 freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace,
1605 prog_fd, "subprog_tc");
1606 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace"))
1607 goto out;
1608
1609 err = bpf_link__destroy(freplace_link);
1610 freplace_link = NULL;
1611 if (!ASSERT_OK(err, "destroy link"))
1612 goto out;
1613
1614 /* OK to update prog_array map then delete element from the map. */
1615
1616 key = 0;
1617 map_fd = bpf_map__fd(freplace_skel->maps.jmp_table);
1618 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1619 if (!ASSERT_OK(err, "update jmp_table"))
1620 goto out;
1621
1622 err = bpf_map_delete_elem(map_fd, &key);
1623 if (!ASSERT_OK(err, "delete_elem from jmp_table"))
1624 goto out;
1625
1626 /* Fail to attach a tail callee prog with freplace prog. */
1627
1628 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1629 if (!ASSERT_OK(err, "update jmp_table"))
1630 goto out;
1631
1632 freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace,
1633 prog_fd, "subprog_tc");
1634 if (!ASSERT_ERR_PTR(freplace_link, "attach_freplace failure"))
1635 goto out;
1636
1637 err = bpf_map_delete_elem(map_fd, &key);
1638 if (!ASSERT_OK(err, "delete_elem from jmp_table"))
1639 goto out;
1640
1641 /* Fail to update an extended prog to prog_array map. */
1642
1643 freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace,
1644 prog_fd, "subprog_tc");
1645 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace"))
1646 goto out;
1647
1648 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1649 if (!ASSERT_ERR(err, "update jmp_table failure"))
1650 goto out;
1651
1652 out:
1653 bpf_link__destroy(freplace_link);
1654 tailcall_freplace__destroy(freplace_skel);
1655 tc_bpf2bpf__destroy(tc_skel);
1656 }
1657
test_tailcall_failure()1658 static void test_tailcall_failure()
1659 {
1660 RUN_TESTS(tailcall_fail);
1661 }
1662
test_tailcall_cgrp_storage(void)1663 static void test_tailcall_cgrp_storage(void)
1664 {
1665 struct tailcall_cgrp_storage_owner *owner_skel = NULL;
1666 struct tailcall_cgrp_storage *skel = NULL;
1667 int err, key = 0, prog_array_fd, prog_fd, storage_map_fd;
1668
1669 owner_skel = tailcall_cgrp_storage_owner__open_and_load();
1670 if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load"))
1671 return;
1672
1673 prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array);
1674 storage_map_fd = bpf_map__fd(owner_skel->maps.storage_map);
1675
1676 skel = tailcall_cgrp_storage__open();
1677 if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage__open"))
1678 goto out;
1679
1680 err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd);
1681 if (!ASSERT_OK(err, "reuse_prog_array"))
1682 goto out;
1683
1684 err = bpf_map__reuse_fd(skel->maps.storage_map, storage_map_fd);
1685 if (!ASSERT_OK(err, "reuse_storage_map"))
1686 goto out;
1687
1688 err = bpf_object__load(skel->obj);
1689 if (!ASSERT_OK(err, "tailcall_cgrp_storage__load"))
1690 goto out;
1691
1692 prog_fd = bpf_program__fd(skel->progs.callee_prog);
1693 err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY);
1694 ASSERT_OK(err, "update_prog_array");
1695 out:
1696 tailcall_cgrp_storage__destroy(skel);
1697 tailcall_cgrp_storage_owner__destroy(owner_skel);
1698 }
1699
test_tailcall_cgrp_storage_diff_storage(void)1700 static void test_tailcall_cgrp_storage_diff_storage(void)
1701 {
1702 struct tailcall_cgrp_storage_owner *owner_skel = NULL;
1703 struct tailcall_cgrp_storage *skel = NULL;
1704 int err, prog_array_fd;
1705
1706 owner_skel = tailcall_cgrp_storage_owner__open_and_load();
1707 if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load"))
1708 return;
1709
1710 prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array);
1711
1712 skel = tailcall_cgrp_storage__open();
1713 if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage__open"))
1714 goto out;
1715
1716 err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd);
1717 if (!ASSERT_OK(err, "reuse_prog_array"))
1718 goto out;
1719
1720 err = bpf_object__load(skel->obj);
1721 ASSERT_ERR(err, "tailcall_cgrp_storage__load");
1722 out:
1723 tailcall_cgrp_storage__destroy(skel);
1724 tailcall_cgrp_storage_owner__destroy(owner_skel);
1725 }
1726
test_tailcall_cgrp_storage_no_storage(void)1727 static void test_tailcall_cgrp_storage_no_storage(void)
1728 {
1729 struct tailcall_cgrp_storage_owner *owner_skel = NULL;
1730 struct tailcall_cgrp_storage_no_storage *skel = NULL;
1731 int err, prog_array_fd;
1732
1733 owner_skel = tailcall_cgrp_storage_owner__open_and_load();
1734 if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load"))
1735 return;
1736
1737 prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array);
1738
1739 skel = tailcall_cgrp_storage_no_storage__open();
1740 if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage_no_storage__open"))
1741 goto out;
1742
1743 err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd);
1744 if (!ASSERT_OK(err, "reuse_prog_array"))
1745 goto out;
1746
1747 err = bpf_object__load(skel->obj);
1748 ASSERT_ERR(err, "tailcall_cgrp_storage_no_storage__load");
1749 out:
1750 tailcall_cgrp_storage_no_storage__destroy(skel);
1751 tailcall_cgrp_storage_owner__destroy(owner_skel);
1752 }
1753
test_tailcall_cgrp_storage_no_storage_leaf(void)1754 static void test_tailcall_cgrp_storage_no_storage_leaf(void)
1755 {
1756 struct tailcall_cgrp_storage_owner *owner_skel = NULL;
1757 struct tailcall_cgrp_storage_no_storage *skel = NULL;
1758 int err, key = 0, prog_array_fd, prog_fd;
1759
1760 owner_skel = tailcall_cgrp_storage_owner__open_and_load();
1761 if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load"))
1762 return;
1763
1764 prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array);
1765
1766 skel = tailcall_cgrp_storage_no_storage__open_and_load();
1767 if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage_no_storage__open_and_load"))
1768 goto out;
1769
1770 prog_fd = bpf_program__fd(skel->progs.leaf_prog);
1771 err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY);
1772 if (!ASSERT_OK(err, "update_prog_array_leaf"))
1773 goto out;
1774
1775 prog_fd = bpf_program__fd(skel->progs.caller_prog);
1776 err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY);
1777 ASSERT_ERR(err, "update_prog_array_bridge");
1778 out:
1779 tailcall_cgrp_storage_no_storage__destroy(skel);
1780 tailcall_cgrp_storage_owner__destroy(owner_skel);
1781 }
1782
test_tailcall_cgrp_storage_no_storage_bridge(void)1783 static void test_tailcall_cgrp_storage_no_storage_bridge(void)
1784 {
1785 struct tailcall_cgrp_storage_owner *owner_skel = NULL;
1786 struct tailcall_cgrp_storage_no_storage *bridge_skel = NULL;
1787 struct tailcall_cgrp_storage *callee_skel = NULL;
1788 int err, key = 0, prog_array_fd, prog_fd, storage_map_fd;
1789
1790 owner_skel = tailcall_cgrp_storage_owner__open_and_load();
1791 if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load"))
1792 return;
1793
1794 prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array);
1795 storage_map_fd = bpf_map__fd(owner_skel->maps.storage_map);
1796
1797 callee_skel = tailcall_cgrp_storage__open();
1798 if (!ASSERT_OK_PTR(callee_skel, "tailcall_cgrp_storage__open"))
1799 goto out;
1800
1801 bpf_program__set_autoload(callee_skel->progs.caller_prog, false);
1802
1803 err = bpf_map__reuse_fd(callee_skel->maps.prog_array, prog_array_fd);
1804 if (!ASSERT_OK(err, "reuse_prog_array"))
1805 goto out;
1806
1807 err = bpf_map__reuse_fd(callee_skel->maps.storage_map, storage_map_fd);
1808 if (!ASSERT_OK(err, "reuse_storage_map"))
1809 goto out;
1810
1811 err = bpf_object__load(callee_skel->obj);
1812 if (!ASSERT_OK(err, "tailcall_cgrp_storage__load"))
1813 goto out;
1814
1815 prog_fd = bpf_program__fd(callee_skel->progs.callee_prog);
1816 err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY);
1817 if (!ASSERT_OK(err, "update_prog_array"))
1818 goto out;
1819
1820 bridge_skel = tailcall_cgrp_storage_no_storage__open();
1821 if (!ASSERT_OK_PTR(bridge_skel, "tailcall_cgrp_storage_no_storage__open"))
1822 goto out;
1823
1824 err = bpf_map__reuse_fd(bridge_skel->maps.prog_array, prog_array_fd);
1825 if (!ASSERT_OK(err, "reuse_prog_array"))
1826 goto out;
1827
1828 err = bpf_object__load(bridge_skel->obj);
1829 ASSERT_ERR(err, "tailcall_cgrp_storage_no_storage_bridge__load");
1830 out:
1831 tailcall_cgrp_storage_no_storage__destroy(bridge_skel);
1832 tailcall_cgrp_storage__destroy(callee_skel);
1833 tailcall_cgrp_storage_owner__destroy(owner_skel);
1834 }
1835
uprobe_sleepable_trigger(void)1836 noinline void uprobe_sleepable_trigger(void)
1837 {
1838 asm volatile ("");
1839 }
1840
test_tailcall_sleepable(void)1841 static void test_tailcall_sleepable(void)
1842 {
1843 LIBBPF_OPTS(bpf_uprobe_opts, opts);
1844 struct tailcall_sleepable *skel;
1845 int prog_fd, map_fd;
1846 int err, key;
1847
1848 skel = tailcall_sleepable__open();
1849 if (!ASSERT_OK_PTR(skel, "tailcall_sleepable__open"))
1850 return;
1851
1852 /*
1853 * Test that we can't load uprobe_normal and uprobe_sleepable_1,
1854 * because they share tailcall map.
1855 */
1856 bpf_program__set_autoload(skel->progs.uprobe_normal, true);
1857 bpf_program__set_autoload(skel->progs.uprobe_sleepable_1, true);
1858
1859 err = tailcall_sleepable__load(skel);
1860 if (!ASSERT_ERR(err, "tailcall_sleepable__load"))
1861 goto out;
1862
1863 tailcall_sleepable__destroy(skel);
1864
1865 /*
1866 * Test that we can tail call from sleepable to sleepable program.
1867 */
1868 skel = tailcall_sleepable__open();
1869 if (!ASSERT_OK_PTR(skel, "tailcall_sleepable__open"))
1870 return;
1871
1872 bpf_program__set_autoload(skel->progs.uprobe_sleepable_1, true);
1873 bpf_program__set_autoload(skel->progs.uprobe_sleepable_2, true);
1874
1875 err = tailcall_sleepable__load(skel);
1876 if (!ASSERT_OK(err, "tailcall_sleepable__load"))
1877 goto out;
1878
1879 /* Add sleepable uprobe_sleepable_2 to jmp_table[0]. */
1880 key = 0;
1881 prog_fd = bpf_program__fd(skel->progs.uprobe_sleepable_2);
1882 map_fd = bpf_map__fd(skel->maps.jmp_table);
1883 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1884 if (!ASSERT_OK(err, "update jmp_table"))
1885 goto out;
1886
1887 skel->bss->my_pid = getpid();
1888
1889 /* Attach uprobe_sleepable_1 to uprobe_sleepable_trigger and hit it. */
1890 opts.func_name = "uprobe_sleepable_trigger";
1891 skel->links.uprobe_sleepable_1 = bpf_program__attach_uprobe_opts(
1892 skel->progs.uprobe_sleepable_1,
1893 -1,
1894 "/proc/self/exe",
1895 0 /* offset */,
1896 &opts);
1897 if (!ASSERT_OK_PTR(skel->links.uprobe_sleepable_1, "bpf_program__attach_uprobe_opts"))
1898 goto out;
1899
1900 uprobe_sleepable_trigger();
1901 ASSERT_EQ(skel->bss->executed, 1, "executed");
1902
1903 out:
1904 tailcall_sleepable__destroy(skel);
1905 }
1906
test_tailcall_callback(void)1907 static void test_tailcall_callback(void)
1908 {
1909 RUN_TESTS(tailcall_callback);
1910 }
1911
test_tailcall_bpf2bpf_fexit_links(void)1912 static void test_tailcall_bpf2bpf_fexit_links(void)
1913 {
1914 struct tailcall_bpf2bpf_fexit *skel1 = NULL, *skel2 = NULL;
1915 struct tailcall_bpf2bpf2 *skel_tc;
1916 int err, prog_fd;
1917
1918 skel_tc = tailcall_bpf2bpf2__open_and_load();
1919 if (!ASSERT_OK_PTR(skel_tc, "tailcall_bpf2bpf2__open_and_load"))
1920 return;
1921
1922 skel1 = tailcall_bpf2bpf_fexit__open();
1923 if (!ASSERT_OK_PTR(skel1, "tailcall_bpf2bpf_fexit__open"))
1924 goto out;
1925
1926 prog_fd = bpf_program__fd(skel_tc->progs.classifier_0);
1927 err = bpf_program__set_attach_target(skel1->progs.fexit, prog_fd, "subprog_tail");
1928 if (!ASSERT_OK(err, "bpf_program__set_attach_target"))
1929 goto out;
1930
1931 err = tailcall_bpf2bpf_fexit__load(skel1);
1932 if (!ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load"))
1933 goto out;
1934
1935 skel1->links.fexit = bpf_program__attach_trace(skel1->progs.fexit);
1936 if (!ASSERT_OK_PTR(skel1->links.fexit, "bpf_program__attach_trace"))
1937 goto out;
1938
1939 skel2 = tailcall_bpf2bpf_fexit__open();
1940 if (!ASSERT_OK_PTR(skel2, "tailcall_bpf2bpf_fexit__open"))
1941 goto out;
1942
1943 err = bpf_program__set_attach_target(skel2->progs.fexit, prog_fd, "subprog_tail");
1944 if (!ASSERT_OK(err, "bpf_program__set_attach_target"))
1945 goto out;
1946
1947 err = tailcall_bpf2bpf_fexit__load(skel2);
1948 ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load");
1949
1950 out:
1951 tailcall_bpf2bpf_fexit__destroy(skel1);
1952 tailcall_bpf2bpf_fexit__destroy(skel2);
1953 tailcall_bpf2bpf2__destroy(skel_tc);
1954 }
1955
test_tailcalls(void)1956 void test_tailcalls(void)
1957 {
1958 if (test__start_subtest("tailcall_1"))
1959 test_tailcall_1();
1960 if (test__start_subtest("tailcall_2"))
1961 test_tailcall_2();
1962 if (test__start_subtest("tailcall_3"))
1963 test_tailcall_3();
1964 if (test__start_subtest("tailcall_4"))
1965 test_tailcall_4();
1966 if (test__start_subtest("tailcall_5"))
1967 test_tailcall_5();
1968 if (test__start_subtest("tailcall_6"))
1969 test_tailcall_6();
1970 if (test__start_subtest("tailcall_bpf2bpf_1"))
1971 test_tailcall_bpf2bpf_1();
1972 if (test__start_subtest("tailcall_bpf2bpf_2"))
1973 test_tailcall_bpf2bpf_2();
1974 if (test__start_subtest("tailcall_bpf2bpf_3"))
1975 test_tailcall_bpf2bpf_3();
1976 if (test__start_subtest("tailcall_bpf2bpf_4"))
1977 test_tailcall_bpf2bpf_4(false);
1978 if (test__start_subtest("tailcall_bpf2bpf_5"))
1979 test_tailcall_bpf2bpf_4(true);
1980 if (test__start_subtest("tailcall_bpf2bpf_6"))
1981 test_tailcall_bpf2bpf_6();
1982 if (test__start_subtest("tailcall_bpf2bpf_fentry"))
1983 test_tailcall_bpf2bpf_fentry();
1984 if (test__start_subtest("tailcall_bpf2bpf_fexit"))
1985 test_tailcall_bpf2bpf_fexit();
1986 if (test__start_subtest("tailcall_bpf2bpf_fentry_fexit"))
1987 test_tailcall_bpf2bpf_fentry_fexit();
1988 if (test__start_subtest("tailcall_bpf2bpf_fentry_entry"))
1989 test_tailcall_bpf2bpf_fentry_entry();
1990 if (test__start_subtest("tailcall_poke"))
1991 test_tailcall_poke();
1992 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_1"))
1993 test_tailcall_bpf2bpf_hierarchy_1();
1994 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fentry"))
1995 test_tailcall_bpf2bpf_hierarchy_fentry();
1996 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fexit"))
1997 test_tailcall_bpf2bpf_hierarchy_fexit();
1998 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fentry_fexit"))
1999 test_tailcall_bpf2bpf_hierarchy_fentry_fexit();
2000 if (test__start_subtest("tailcall_bpf2bpf_hierarchy_fentry_entry"))
2001 test_tailcall_bpf2bpf_hierarchy_fentry_entry();
2002 test_tailcall_bpf2bpf_hierarchy_2();
2003 test_tailcall_bpf2bpf_hierarchy_3();
2004 if (test__start_subtest("tailcall_freplace"))
2005 test_tailcall_freplace();
2006 if (test__start_subtest("tailcall_bpf2bpf_freplace"))
2007 test_tailcall_bpf2bpf_freplace();
2008 if (test__start_subtest("tailcall_failure"))
2009 test_tailcall_failure();
2010 if (test__start_subtest("tailcall_sleepable"))
2011 test_tailcall_sleepable();
2012 if (test__start_subtest("tailcall_cgrp_storage"))
2013 test_tailcall_cgrp_storage();
2014 if (test__start_subtest("tailcall_cgrp_storage_diff_storage"))
2015 test_tailcall_cgrp_storage_diff_storage();
2016 if (test__start_subtest("tailcall_cgrp_storage_no_storage"))
2017 test_tailcall_cgrp_storage_no_storage();
2018 if (test__start_subtest("tailcall_cgrp_storage_no_storage_leaf"))
2019 test_tailcall_cgrp_storage_no_storage_leaf();
2020 if (test__start_subtest("tailcall_cgrp_storage_no_storage_bridge"))
2021 test_tailcall_cgrp_storage_no_storage_bridge();
2022 test_tailcall_callback();
2023 if (test__start_subtest("tailcall_bpf2bpf_fexit_links"))
2024 test_tailcall_bpf2bpf_fexit_links();
2025 }
2026