1 // SPDX-License-Identifier: GPL-2.0
2 #include <test_progs.h>
3 #include "kprobe_multi.skel.h"
4 #include "trace_helpers.h"
5 #include "kprobe_multi_empty.skel.h"
6 #include "kprobe_multi_override.skel.h"
7 #include "kprobe_multi_session.skel.h"
8 #include "kprobe_multi_session_cookie.skel.h"
9 #include "kprobe_multi_verifier.skel.h"
10 #include "bpf/libbpf_internal.h"
11 #include "bpf/hashmap.h"
12
kprobe_multi_test_run(struct kprobe_multi * skel,bool test_return)13 static void kprobe_multi_test_run(struct kprobe_multi *skel, bool test_return)
14 {
15 LIBBPF_OPTS(bpf_test_run_opts, topts);
16 int err, prog_fd;
17
18 prog_fd = bpf_program__fd(skel->progs.trigger);
19 err = bpf_prog_test_run_opts(prog_fd, &topts);
20 ASSERT_OK(err, "test_run");
21 ASSERT_EQ(topts.retval, 0, "test_run");
22
23 ASSERT_EQ(skel->bss->kprobe_test1_result, 1, "kprobe_test1_result");
24 ASSERT_EQ(skel->bss->kprobe_test2_result, 1, "kprobe_test2_result");
25 ASSERT_EQ(skel->bss->kprobe_test3_result, 1, "kprobe_test3_result");
26 ASSERT_EQ(skel->bss->kprobe_test4_result, 1, "kprobe_test4_result");
27 ASSERT_EQ(skel->bss->kprobe_test5_result, 1, "kprobe_test5_result");
28 ASSERT_EQ(skel->bss->kprobe_test6_result, 1, "kprobe_test6_result");
29 ASSERT_EQ(skel->bss->kprobe_test7_result, 1, "kprobe_test7_result");
30 ASSERT_EQ(skel->bss->kprobe_test8_result, 1, "kprobe_test8_result");
31
32 if (test_return) {
33 ASSERT_EQ(skel->bss->kretprobe_test1_result, 1, "kretprobe_test1_result");
34 ASSERT_EQ(skel->bss->kretprobe_test2_result, 1, "kretprobe_test2_result");
35 ASSERT_EQ(skel->bss->kretprobe_test3_result, 1, "kretprobe_test3_result");
36 ASSERT_EQ(skel->bss->kretprobe_test4_result, 1, "kretprobe_test4_result");
37 ASSERT_EQ(skel->bss->kretprobe_test5_result, 1, "kretprobe_test5_result");
38 ASSERT_EQ(skel->bss->kretprobe_test6_result, 1, "kretprobe_test6_result");
39 ASSERT_EQ(skel->bss->kretprobe_test7_result, 1, "kretprobe_test7_result");
40 ASSERT_EQ(skel->bss->kretprobe_test8_result, 1, "kretprobe_test8_result");
41 }
42 }
43
test_skel_api(void)44 static void test_skel_api(void)
45 {
46 struct kprobe_multi *skel = NULL;
47 int err;
48
49 skel = kprobe_multi__open_and_load();
50 if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load"))
51 goto cleanup;
52
53 skel->bss->pid = getpid();
54 err = kprobe_multi__attach(skel);
55 if (!ASSERT_OK(err, "kprobe_multi__attach"))
56 goto cleanup;
57
58 kprobe_multi_test_run(skel, true);
59
60 cleanup:
61 kprobe_multi__destroy(skel);
62 }
63
test_link_api(struct bpf_link_create_opts * opts)64 static void test_link_api(struct bpf_link_create_opts *opts)
65 {
66 int prog_fd, link1_fd = -1, link2_fd = -1;
67 struct kprobe_multi *skel = NULL;
68
69 skel = kprobe_multi__open_and_load();
70 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
71 goto cleanup;
72
73 skel->bss->pid = getpid();
74 prog_fd = bpf_program__fd(skel->progs.test_kprobe);
75 link1_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
76 if (!ASSERT_GE(link1_fd, 0, "link_fd"))
77 goto cleanup;
78
79 opts->kprobe_multi.flags = BPF_F_KPROBE_MULTI_RETURN;
80 prog_fd = bpf_program__fd(skel->progs.test_kretprobe);
81 link2_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
82 if (!ASSERT_GE(link2_fd, 0, "link_fd"))
83 goto cleanup;
84
85 kprobe_multi_test_run(skel, true);
86
87 cleanup:
88 if (link1_fd != -1)
89 close(link1_fd);
90 if (link2_fd != -1)
91 close(link2_fd);
92 kprobe_multi__destroy(skel);
93 }
94
95 #define GET_ADDR(__sym, __addr) ({ \
96 __addr = ksym_get_addr(__sym); \
97 if (!ASSERT_NEQ(__addr, 0, "kallsyms load failed for " #__sym)) \
98 return; \
99 })
100
test_link_api_addrs(void)101 static void test_link_api_addrs(void)
102 {
103 LIBBPF_OPTS(bpf_link_create_opts, opts);
104 unsigned long long addrs[8];
105
106 GET_ADDR("bpf_fentry_test1", addrs[0]);
107 GET_ADDR("bpf_fentry_test2", addrs[1]);
108 GET_ADDR("bpf_fentry_test3", addrs[2]);
109 GET_ADDR("bpf_fentry_test4", addrs[3]);
110 GET_ADDR("bpf_fentry_test5", addrs[4]);
111 GET_ADDR("bpf_fentry_test6", addrs[5]);
112 GET_ADDR("bpf_fentry_test7", addrs[6]);
113 GET_ADDR("bpf_fentry_test8", addrs[7]);
114
115 opts.kprobe_multi.addrs = (const unsigned long*) addrs;
116 opts.kprobe_multi.cnt = ARRAY_SIZE(addrs);
117 test_link_api(&opts);
118 }
119
test_link_api_syms(void)120 static void test_link_api_syms(void)
121 {
122 LIBBPF_OPTS(bpf_link_create_opts, opts);
123 const char *syms[8] = {
124 "bpf_fentry_test1",
125 "bpf_fentry_test2",
126 "bpf_fentry_test3",
127 "bpf_fentry_test4",
128 "bpf_fentry_test5",
129 "bpf_fentry_test6",
130 "bpf_fentry_test7",
131 "bpf_fentry_test8",
132 };
133
134 opts.kprobe_multi.syms = syms;
135 opts.kprobe_multi.cnt = ARRAY_SIZE(syms);
136 test_link_api(&opts);
137 }
138
139 static void
test_attach_api(const char * pattern,struct bpf_kprobe_multi_opts * opts)140 test_attach_api(const char *pattern, struct bpf_kprobe_multi_opts *opts)
141 {
142 struct bpf_link *link1 = NULL, *link2 = NULL;
143 struct kprobe_multi *skel = NULL;
144
145 skel = kprobe_multi__open_and_load();
146 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
147 goto cleanup;
148
149 skel->bss->pid = getpid();
150 link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
151 pattern, opts);
152 if (!ASSERT_OK_PTR(link1, "bpf_program__attach_kprobe_multi_opts"))
153 goto cleanup;
154
155 if (opts) {
156 opts->retprobe = true;
157 link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual,
158 pattern, opts);
159 if (!ASSERT_OK_PTR(link2, "bpf_program__attach_kprobe_multi_opts"))
160 goto cleanup;
161 }
162
163 kprobe_multi_test_run(skel, !!opts);
164
165 cleanup:
166 bpf_link__destroy(link2);
167 bpf_link__destroy(link1);
168 kprobe_multi__destroy(skel);
169 }
170
test_attach_api_pattern(void)171 static void test_attach_api_pattern(void)
172 {
173 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
174
175 test_attach_api("bpf_fentry_test*", &opts);
176 test_attach_api("bpf_fentry_test?", NULL);
177 }
178
test_attach_api_addrs(void)179 static void test_attach_api_addrs(void)
180 {
181 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
182 unsigned long long addrs[8];
183
184 GET_ADDR("bpf_fentry_test1", addrs[0]);
185 GET_ADDR("bpf_fentry_test2", addrs[1]);
186 GET_ADDR("bpf_fentry_test3", addrs[2]);
187 GET_ADDR("bpf_fentry_test4", addrs[3]);
188 GET_ADDR("bpf_fentry_test5", addrs[4]);
189 GET_ADDR("bpf_fentry_test6", addrs[5]);
190 GET_ADDR("bpf_fentry_test7", addrs[6]);
191 GET_ADDR("bpf_fentry_test8", addrs[7]);
192
193 opts.addrs = (const unsigned long *) addrs;
194 opts.cnt = ARRAY_SIZE(addrs);
195 test_attach_api(NULL, &opts);
196 }
197
test_attach_api_syms(void)198 static void test_attach_api_syms(void)
199 {
200 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
201 const char *syms[8] = {
202 "bpf_fentry_test1",
203 "bpf_fentry_test2",
204 "bpf_fentry_test3",
205 "bpf_fentry_test4",
206 "bpf_fentry_test5",
207 "bpf_fentry_test6",
208 "bpf_fentry_test7",
209 "bpf_fentry_test8",
210 };
211
212 opts.syms = syms;
213 opts.cnt = ARRAY_SIZE(syms);
214 test_attach_api(NULL, &opts);
215 }
216
test_attach_api_fails(void)217 static void test_attach_api_fails(void)
218 {
219 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
220 struct kprobe_multi *skel = NULL;
221 struct bpf_link *link = NULL;
222 unsigned long long addrs[2];
223 const char *syms[2] = {
224 "bpf_fentry_test1",
225 "bpf_fentry_test2",
226 };
227 __u64 cookies[2];
228 int saved_error;
229
230 addrs[0] = ksym_get_addr("bpf_fentry_test1");
231 addrs[1] = ksym_get_addr("bpf_fentry_test2");
232
233 if (!ASSERT_FALSE(!addrs[0] || !addrs[1], "ksym_get_addr"))
234 goto cleanup;
235
236 skel = kprobe_multi__open_and_load();
237 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
238 goto cleanup;
239
240 skel->bss->pid = getpid();
241
242 /* fail_1 - pattern and opts NULL */
243 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
244 NULL, NULL);
245 saved_error = -errno;
246 if (!ASSERT_ERR_PTR(link, "fail_1"))
247 goto cleanup;
248
249 if (!ASSERT_EQ(saved_error, -EINVAL, "fail_1_error"))
250 goto cleanup;
251
252 /* fail_2 - both addrs and syms set */
253 opts.addrs = (const unsigned long *) addrs;
254 opts.syms = syms;
255 opts.cnt = ARRAY_SIZE(syms);
256 opts.cookies = NULL;
257
258 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
259 NULL, &opts);
260 saved_error = -errno;
261 if (!ASSERT_ERR_PTR(link, "fail_2"))
262 goto cleanup;
263
264 if (!ASSERT_EQ(saved_error, -EINVAL, "fail_2_error"))
265 goto cleanup;
266
267 /* fail_3 - pattern and addrs set */
268 opts.addrs = (const unsigned long *) addrs;
269 opts.syms = NULL;
270 opts.cnt = ARRAY_SIZE(syms);
271 opts.cookies = NULL;
272
273 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
274 "ksys_*", &opts);
275 saved_error = -errno;
276 if (!ASSERT_ERR_PTR(link, "fail_3"))
277 goto cleanup;
278
279 if (!ASSERT_EQ(saved_error, -EINVAL, "fail_3_error"))
280 goto cleanup;
281
282 /* fail_4 - pattern and cnt set */
283 opts.addrs = NULL;
284 opts.syms = NULL;
285 opts.cnt = ARRAY_SIZE(syms);
286 opts.cookies = NULL;
287
288 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
289 "ksys_*", &opts);
290 saved_error = -errno;
291 if (!ASSERT_ERR_PTR(link, "fail_4"))
292 goto cleanup;
293
294 if (!ASSERT_EQ(saved_error, -EINVAL, "fail_4_error"))
295 goto cleanup;
296
297 /* fail_5 - pattern and cookies */
298 opts.addrs = NULL;
299 opts.syms = NULL;
300 opts.cnt = 0;
301 opts.cookies = cookies;
302
303 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
304 "ksys_*", &opts);
305 saved_error = -errno;
306 if (!ASSERT_ERR_PTR(link, "fail_5"))
307 goto cleanup;
308
309 if (!ASSERT_EQ(saved_error, -EINVAL, "fail_5_error"))
310 goto cleanup;
311
312 /* fail_6 - abnormal cnt */
313 opts.addrs = (const unsigned long *) addrs;
314 opts.syms = NULL;
315 opts.cnt = INT_MAX;
316 opts.cookies = NULL;
317
318 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
319 NULL, &opts);
320 saved_error = -errno;
321 if (!ASSERT_ERR_PTR(link, "fail_6"))
322 goto cleanup;
323
324 if (!ASSERT_EQ(saved_error, -E2BIG, "fail_6_error"))
325 goto cleanup;
326
327 cleanup:
328 bpf_link__destroy(link);
329 kprobe_multi__destroy(skel);
330 }
331
test_session_skel_api(void)332 static void test_session_skel_api(void)
333 {
334 struct kprobe_multi_session *skel = NULL;
335 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
336 LIBBPF_OPTS(bpf_test_run_opts, topts);
337 struct bpf_link *link = NULL;
338 int i, err, prog_fd;
339
340 skel = kprobe_multi_session__open_and_load();
341 if (!ASSERT_OK_PTR(skel, "kprobe_multi_session__open_and_load"))
342 return;
343
344 skel->bss->pid = getpid();
345
346 err = kprobe_multi_session__attach(skel);
347 if (!ASSERT_OK(err, " kprobe_multi_session__attach"))
348 goto cleanup;
349
350 prog_fd = bpf_program__fd(skel->progs.trigger);
351 err = bpf_prog_test_run_opts(prog_fd, &topts);
352 ASSERT_OK(err, "test_run");
353 ASSERT_EQ(topts.retval, 0, "test_run");
354
355 /* bpf_fentry_test1-4 trigger return probe, result is 2 */
356 for (i = 0; i < 4; i++)
357 ASSERT_EQ(skel->bss->kprobe_session_result[i], 2, "kprobe_session_result");
358
359 /* bpf_fentry_test5-8 trigger only entry probe, result is 1 */
360 for (i = 4; i < 8; i++)
361 ASSERT_EQ(skel->bss->kprobe_session_result[i], 1, "kprobe_session_result");
362
363 cleanup:
364 bpf_link__destroy(link);
365 kprobe_multi_session__destroy(skel);
366 }
367
test_session_cookie_skel_api(void)368 static void test_session_cookie_skel_api(void)
369 {
370 struct kprobe_multi_session_cookie *skel = NULL;
371 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
372 LIBBPF_OPTS(bpf_test_run_opts, topts);
373 struct bpf_link *link = NULL;
374 int err, prog_fd;
375
376 skel = kprobe_multi_session_cookie__open_and_load();
377 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
378 return;
379
380 skel->bss->pid = getpid();
381
382 err = kprobe_multi_session_cookie__attach(skel);
383 if (!ASSERT_OK(err, " kprobe_multi_wrapper__attach"))
384 goto cleanup;
385
386 prog_fd = bpf_program__fd(skel->progs.trigger);
387 err = bpf_prog_test_run_opts(prog_fd, &topts);
388 ASSERT_OK(err, "test_run");
389 ASSERT_EQ(topts.retval, 0, "test_run");
390
391 ASSERT_EQ(skel->bss->test_kprobe_1_result, 1, "test_kprobe_1_result");
392 ASSERT_EQ(skel->bss->test_kprobe_2_result, 2, "test_kprobe_2_result");
393 ASSERT_EQ(skel->bss->test_kprobe_3_result, 3, "test_kprobe_3_result");
394
395 cleanup:
396 bpf_link__destroy(link);
397 kprobe_multi_session_cookie__destroy(skel);
398 }
399
test_unique_match(void)400 static void test_unique_match(void)
401 {
402 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
403 struct kprobe_multi *skel = NULL;
404 struct bpf_link *link = NULL;
405
406 skel = kprobe_multi__open_and_load();
407 if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load"))
408 return;
409
410 opts.unique_match = true;
411 skel->bss->pid = getpid();
412 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
413 "bpf_fentry_test*", &opts);
414 if (!ASSERT_ERR_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
415 bpf_link__destroy(link);
416
417 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
418 "bpf_fentry_test8*", &opts);
419 if (ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
420 bpf_link__destroy(link);
421
422 kprobe_multi__destroy(skel);
423 }
424
symbol_hash(long key,void * ctx __maybe_unused)425 static size_t symbol_hash(long key, void *ctx __maybe_unused)
426 {
427 return str_hash((const char *) key);
428 }
429
symbol_equal(long key1,long key2,void * ctx __maybe_unused)430 static bool symbol_equal(long key1, long key2, void *ctx __maybe_unused)
431 {
432 return strcmp((const char *) key1, (const char *) key2) == 0;
433 }
434
is_invalid_entry(char * buf,bool kernel)435 static bool is_invalid_entry(char *buf, bool kernel)
436 {
437 if (kernel && strchr(buf, '['))
438 return true;
439 if (!kernel && !strchr(buf, '['))
440 return true;
441 return false;
442 }
443
skip_entry(char * name)444 static bool skip_entry(char *name)
445 {
446 /*
447 * We attach to almost all kernel functions and some of them
448 * will cause 'suspicious RCU usage' when fprobe is attached
449 * to them. Filter out the current culprits - arch_cpu_idle
450 * default_idle and rcu_* functions.
451 */
452 if (!strcmp(name, "arch_cpu_idle"))
453 return true;
454 if (!strcmp(name, "default_idle"))
455 return true;
456 if (!strncmp(name, "rcu_", 4))
457 return true;
458 if (!strcmp(name, "bpf_dispatcher_xdp_func"))
459 return true;
460 if (!strncmp(name, "__ftrace_invalid_address__",
461 sizeof("__ftrace_invalid_address__") - 1))
462 return true;
463 return false;
464 }
465
466 /* Do comparision by ignoring '.llvm.<hash>' suffixes. */
compare_name(const char * name1,const char * name2)467 static int compare_name(const char *name1, const char *name2)
468 {
469 const char *res1, *res2;
470 int len1, len2;
471
472 res1 = strstr(name1, ".llvm.");
473 res2 = strstr(name2, ".llvm.");
474 len1 = res1 ? res1 - name1 : strlen(name1);
475 len2 = res2 ? res2 - name2 : strlen(name2);
476
477 if (len1 == len2)
478 return strncmp(name1, name2, len1);
479 if (len1 < len2)
480 return strncmp(name1, name2, len1) <= 0 ? -1 : 1;
481 return strncmp(name1, name2, len2) >= 0 ? 1 : -1;
482 }
483
load_kallsyms_compare(const void * p1,const void * p2)484 static int load_kallsyms_compare(const void *p1, const void *p2)
485 {
486 return compare_name(((const struct ksym *)p1)->name, ((const struct ksym *)p2)->name);
487 }
488
search_kallsyms_compare(const void * p1,const struct ksym * p2)489 static int search_kallsyms_compare(const void *p1, const struct ksym *p2)
490 {
491 return compare_name(p1, p2->name);
492 }
493
get_syms(char *** symsp,size_t * cntp,bool kernel)494 static int get_syms(char ***symsp, size_t *cntp, bool kernel)
495 {
496 size_t cap = 0, cnt = 0;
497 char *name = NULL, *ksym_name, **syms = NULL;
498 struct hashmap *map;
499 struct ksyms *ksyms;
500 struct ksym *ks;
501 char buf[256];
502 FILE *f;
503 int err = 0;
504
505 ksyms = load_kallsyms_custom_local(load_kallsyms_compare);
506 if (!ASSERT_OK_PTR(ksyms, "load_kallsyms_custom_local"))
507 return -EINVAL;
508
509 /*
510 * The available_filter_functions contains many duplicates,
511 * but other than that all symbols are usable in kprobe multi
512 * interface.
513 * Filtering out duplicates by using hashmap__add, which won't
514 * add existing entry.
515 */
516
517 if (access("/sys/kernel/tracing/trace", F_OK) == 0)
518 f = fopen("/sys/kernel/tracing/available_filter_functions", "r");
519 else
520 f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r");
521
522 if (!f)
523 return -EINVAL;
524
525 map = hashmap__new(symbol_hash, symbol_equal, NULL);
526 if (IS_ERR(map)) {
527 err = libbpf_get_error(map);
528 goto error;
529 }
530
531 while (fgets(buf, sizeof(buf), f)) {
532 if (is_invalid_entry(buf, kernel))
533 continue;
534
535 free(name);
536 if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1)
537 continue;
538 if (skip_entry(name))
539 continue;
540
541 ks = search_kallsyms_custom_local(ksyms, name, search_kallsyms_compare);
542 if (!ks) {
543 err = -EINVAL;
544 goto error;
545 }
546
547 ksym_name = ks->name;
548 err = hashmap__add(map, ksym_name, 0);
549 if (err == -EEXIST) {
550 err = 0;
551 continue;
552 }
553 if (err)
554 goto error;
555
556 err = libbpf_ensure_mem((void **) &syms, &cap,
557 sizeof(*syms), cnt + 1);
558 if (err)
559 goto error;
560
561 syms[cnt++] = ksym_name;
562 }
563
564 *symsp = syms;
565 *cntp = cnt;
566
567 error:
568 free(name);
569 fclose(f);
570 hashmap__free(map);
571 if (err)
572 free(syms);
573 return err;
574 }
575
get_addrs(unsigned long ** addrsp,size_t * cntp,bool kernel)576 static int get_addrs(unsigned long **addrsp, size_t *cntp, bool kernel)
577 {
578 unsigned long *addr, *addrs, *tmp_addrs;
579 int err = 0, max_cnt, inc_cnt;
580 char *name = NULL;
581 size_t cnt = 0;
582 char buf[256];
583 FILE *f;
584
585 if (access("/sys/kernel/tracing/trace", F_OK) == 0)
586 f = fopen("/sys/kernel/tracing/available_filter_functions_addrs", "r");
587 else
588 f = fopen("/sys/kernel/debug/tracing/available_filter_functions_addrs", "r");
589
590 if (!f)
591 return -ENOENT;
592
593 /* In my local setup, the number of entries is 50k+ so Let us initially
594 * allocate space to hold 64k entries. If 64k is not enough, incrementally
595 * increase 1k each time.
596 */
597 max_cnt = 65536;
598 inc_cnt = 1024;
599 addrs = malloc(max_cnt * sizeof(long));
600 if (addrs == NULL) {
601 err = -ENOMEM;
602 goto error;
603 }
604
605 while (fgets(buf, sizeof(buf), f)) {
606 if (is_invalid_entry(buf, kernel))
607 continue;
608
609 free(name);
610 if (sscanf(buf, "%p %ms$*[^\n]\n", &addr, &name) != 2)
611 continue;
612 if (skip_entry(name))
613 continue;
614
615 if (cnt == max_cnt) {
616 max_cnt += inc_cnt;
617 tmp_addrs = realloc(addrs, max_cnt);
618 if (!tmp_addrs) {
619 err = -ENOMEM;
620 goto error;
621 }
622 addrs = tmp_addrs;
623 }
624
625 addrs[cnt++] = (unsigned long)addr;
626 }
627
628 *addrsp = addrs;
629 *cntp = cnt;
630
631 error:
632 free(name);
633 fclose(f);
634 if (err)
635 free(addrs);
636 return err;
637 }
638
do_bench_test(struct kprobe_multi_empty * skel,struct bpf_kprobe_multi_opts * opts)639 static void do_bench_test(struct kprobe_multi_empty *skel, struct bpf_kprobe_multi_opts *opts)
640 {
641 long attach_start_ns, attach_end_ns;
642 long detach_start_ns, detach_end_ns;
643 double attach_delta, detach_delta;
644 struct bpf_link *link = NULL;
645
646 attach_start_ns = get_time_ns();
647 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_empty,
648 NULL, opts);
649 attach_end_ns = get_time_ns();
650
651 if (!ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
652 return;
653
654 detach_start_ns = get_time_ns();
655 bpf_link__destroy(link);
656 detach_end_ns = get_time_ns();
657
658 attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0;
659 detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0;
660
661 printf("%s: found %lu functions\n", __func__, opts->cnt);
662 printf("%s: attached in %7.3lfs\n", __func__, attach_delta);
663 printf("%s: detached in %7.3lfs\n", __func__, detach_delta);
664 }
665
test_kprobe_multi_bench_attach(bool kernel)666 static void test_kprobe_multi_bench_attach(bool kernel)
667 {
668 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
669 struct kprobe_multi_empty *skel = NULL;
670 char **syms = NULL;
671 size_t cnt = 0;
672
673 if (!ASSERT_OK(get_syms(&syms, &cnt, kernel), "get_syms"))
674 return;
675
676 skel = kprobe_multi_empty__open_and_load();
677 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
678 goto cleanup;
679
680 opts.syms = (const char **) syms;
681 opts.cnt = cnt;
682
683 do_bench_test(skel, &opts);
684
685 cleanup:
686 kprobe_multi_empty__destroy(skel);
687 if (syms)
688 free(syms);
689 }
690
test_kprobe_multi_bench_attach_addr(bool kernel)691 static void test_kprobe_multi_bench_attach_addr(bool kernel)
692 {
693 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
694 struct kprobe_multi_empty *skel = NULL;
695 unsigned long *addrs = NULL;
696 size_t cnt = 0;
697 int err;
698
699 err = get_addrs(&addrs, &cnt, kernel);
700 if (err == -ENOENT) {
701 test__skip();
702 return;
703 }
704
705 if (!ASSERT_OK(err, "get_addrs"))
706 return;
707
708 skel = kprobe_multi_empty__open_and_load();
709 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
710 goto cleanup;
711
712 opts.addrs = addrs;
713 opts.cnt = cnt;
714
715 do_bench_test(skel, &opts);
716
717 cleanup:
718 kprobe_multi_empty__destroy(skel);
719 free(addrs);
720 }
721
test_attach_override(void)722 static void test_attach_override(void)
723 {
724 struct kprobe_multi_override *skel = NULL;
725 struct bpf_link *link = NULL;
726
727 skel = kprobe_multi_override__open_and_load();
728 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
729 goto cleanup;
730
731 /* The test_override calls bpf_override_return so it should fail
732 * to attach to bpf_fentry_test1 function, which is not on error
733 * injection list.
734 */
735 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_override,
736 "bpf_fentry_test1", NULL);
737 if (!ASSERT_ERR_PTR(link, "override_attached_bpf_fentry_test1")) {
738 bpf_link__destroy(link);
739 goto cleanup;
740 }
741
742 /* The should_fail_bio function is on error injection list,
743 * attach should succeed.
744 */
745 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_override,
746 "should_fail_bio", NULL);
747 if (!ASSERT_OK_PTR(link, "override_attached_should_fail_bio"))
748 goto cleanup;
749
750 bpf_link__destroy(link);
751
752 cleanup:
753 kprobe_multi_override__destroy(skel);
754 }
755
serial_test_kprobe_multi_bench_attach(void)756 void serial_test_kprobe_multi_bench_attach(void)
757 {
758 if (test__start_subtest("kernel"))
759 test_kprobe_multi_bench_attach(true);
760 if (test__start_subtest("modules"))
761 test_kprobe_multi_bench_attach(false);
762 if (test__start_subtest("kernel"))
763 test_kprobe_multi_bench_attach_addr(true);
764 if (test__start_subtest("modules"))
765 test_kprobe_multi_bench_attach_addr(false);
766 }
767
test_kprobe_multi_test(void)768 void test_kprobe_multi_test(void)
769 {
770 if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
771 return;
772
773 if (test__start_subtest("skel_api"))
774 test_skel_api();
775 if (test__start_subtest("link_api_addrs"))
776 test_link_api_syms();
777 if (test__start_subtest("link_api_syms"))
778 test_link_api_addrs();
779 if (test__start_subtest("attach_api_pattern"))
780 test_attach_api_pattern();
781 if (test__start_subtest("attach_api_addrs"))
782 test_attach_api_addrs();
783 if (test__start_subtest("attach_api_syms"))
784 test_attach_api_syms();
785 if (test__start_subtest("attach_api_fails"))
786 test_attach_api_fails();
787 if (test__start_subtest("attach_override"))
788 test_attach_override();
789 if (test__start_subtest("session"))
790 test_session_skel_api();
791 if (test__start_subtest("session_cookie"))
792 test_session_cookie_skel_api();
793 if (test__start_subtest("unique_match"))
794 test_unique_match();
795 RUN_TESTS(kprobe_multi_verifier);
796 }
797