1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3
4 #include <vmlinux.h>
5 #include <bpf/bpf_tracing.h>
6 #include <bpf/bpf_helpers.h>
7
8 #include "../bpf_experimental.h"
9 #include "bpf_misc.h"
10 #include "task_kfunc_common.h"
11
12 char _license[] SEC("license") = "GPL";
13
14 int err, pid;
15
16 /* Prototype for all of the program trace events below:
17 *
18 * TRACE_EVENT(task_newtask,
19 * TP_PROTO(struct task_struct *p, u64 clone_flags)
20 */
21
22 struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym __weak;
23
24 struct task_struct *bpf_task_acquire___one(struct task_struct *task) __ksym __weak;
25 /* The two-param bpf_task_acquire doesn't exist */
26 struct task_struct *bpf_task_acquire___two(struct task_struct *p, void *ctx) __ksym __weak;
27 /* Incorrect type for first param */
28 struct task_struct *bpf_task_acquire___three(void *ctx) __ksym __weak;
29
30 void invalid_kfunc(void) __ksym __weak;
31 void bpf_testmod_test_mod_kfunc(int i) __ksym __weak;
32
is_test_kfunc_task(void)33 static bool is_test_kfunc_task(void)
34 {
35 int cur_pid = bpf_get_current_pid_tgid() >> 32;
36
37 return pid == cur_pid;
38 }
39
test_acquire_release(struct task_struct * task)40 static int test_acquire_release(struct task_struct *task)
41 {
42 struct task_struct *acquired = NULL;
43
44 if (!bpf_ksym_exists(bpf_task_acquire)) {
45 err = 3;
46 return 0;
47 }
48 if (!bpf_ksym_exists(bpf_testmod_test_mod_kfunc)) {
49 err = 4;
50 return 0;
51 }
52 if (bpf_ksym_exists(invalid_kfunc)) {
53 /* the verifier's dead code elimination should remove this */
54 err = 5;
55 asm volatile ("goto -1"); /* for (;;); */
56 }
57
58 acquired = bpf_task_acquire(task);
59 if (acquired)
60 bpf_task_release(acquired);
61 else
62 err = 6;
63
64 return 0;
65 }
66
67 SEC("tp_btf/task_newtask")
BPF_PROG(test_task_kfunc_flavor_relo,struct task_struct * task,u64 clone_flags)68 int BPF_PROG(test_task_kfunc_flavor_relo, struct task_struct *task, u64 clone_flags)
69 {
70 struct task_struct *acquired = NULL;
71 int fake_ctx = 42;
72
73 if (bpf_ksym_exists(bpf_task_acquire___one)) {
74 acquired = bpf_task_acquire___one(task);
75 } else if (bpf_ksym_exists(bpf_task_acquire___two)) {
76 /* Here, bpf_object__resolve_ksym_func_btf_id's find_ksym_btf_id
77 * call will find vmlinux's bpf_task_acquire, but subsequent
78 * bpf_core_types_are_compat will fail
79 */
80 acquired = bpf_task_acquire___two(task, &fake_ctx);
81 err = 3;
82 return 0;
83 } else if (bpf_ksym_exists(bpf_task_acquire___three)) {
84 /* bpf_core_types_are_compat will fail similarly to above case */
85 acquired = bpf_task_acquire___three(&fake_ctx);
86 err = 4;
87 return 0;
88 }
89
90 if (acquired)
91 bpf_task_release(acquired);
92 else
93 err = 5;
94 return 0;
95 }
96
97 SEC("tp_btf/task_newtask")
BPF_PROG(test_task_kfunc_flavor_relo_not_found,struct task_struct * task,u64 clone_flags)98 int BPF_PROG(test_task_kfunc_flavor_relo_not_found, struct task_struct *task, u64 clone_flags)
99 {
100 /* Neither symbol should successfully resolve.
101 * Success or failure of one ___flavor should not affect others
102 */
103 if (bpf_ksym_exists(bpf_task_acquire___two))
104 err = 1;
105 else if (bpf_ksym_exists(bpf_task_acquire___three))
106 err = 2;
107
108 return 0;
109 }
110
111 SEC("tp_btf/task_newtask")
BPF_PROG(test_task_acquire_release_argument,struct task_struct * task,u64 clone_flags)112 int BPF_PROG(test_task_acquire_release_argument, struct task_struct *task, u64 clone_flags)
113 {
114 if (!is_test_kfunc_task())
115 return 0;
116
117 return test_acquire_release(task);
118 }
119
120 SEC("tp_btf/task_newtask")
BPF_PROG(test_task_acquire_release_current,struct task_struct * task,u64 clone_flags)121 int BPF_PROG(test_task_acquire_release_current, struct task_struct *task, u64 clone_flags)
122 {
123 if (!is_test_kfunc_task())
124 return 0;
125
126 return test_acquire_release(bpf_get_current_task_btf());
127 }
128
129 SEC("tp_btf/task_newtask")
BPF_PROG(test_task_acquire_leave_in_map,struct task_struct * task,u64 clone_flags)130 int BPF_PROG(test_task_acquire_leave_in_map, struct task_struct *task, u64 clone_flags)
131 {
132 long status;
133
134 if (!is_test_kfunc_task())
135 return 0;
136
137 status = tasks_kfunc_map_insert(task);
138 if (status)
139 err = 1;
140
141 return 0;
142 }
143
144 SEC("syscall")
test_task_xchg_release(const void * ctx)145 int test_task_xchg_release(const void *ctx)
146 {
147 struct task_struct *task, *kptr, *acquired;
148 struct __tasks_kfunc_map_value *v, *local;
149 int refcnt, refcnt_after_drop;
150 long status;
151
152 (void)ctx;
153
154 task = bpf_get_current_task_btf();
155 status = tasks_kfunc_map_insert(task);
156 if (status) {
157 err = 1;
158 return 0;
159 }
160
161 v = tasks_kfunc_map_value_lookup(task);
162 if (!v) {
163 err = 2;
164 return 0;
165 }
166
167 kptr = bpf_kptr_xchg(&v->task, NULL);
168 if (!kptr) {
169 err = 3;
170 return 0;
171 }
172
173 local = bpf_obj_new(typeof(*local));
174 if (!local) {
175 err = 4;
176 bpf_task_release(kptr);
177 return 0;
178 }
179
180 kptr = bpf_kptr_xchg(&local->task, kptr);
181 if (kptr) {
182 err = 5;
183 bpf_obj_drop(local);
184 bpf_task_release(kptr);
185 return 0;
186 }
187
188 kptr = bpf_kptr_xchg(&local->task, NULL);
189 if (!kptr) {
190 err = 6;
191 bpf_obj_drop(local);
192 return 0;
193 }
194
195 /* Stash a copy into local kptr and check if it is released recursively. */
196 acquired = bpf_task_acquire(kptr);
197 if (!acquired) {
198 err = 7;
199 bpf_obj_drop(local);
200 bpf_task_release(kptr);
201 return 0;
202 }
203 bpf_probe_read_kernel(&refcnt, sizeof(refcnt), &acquired->rcu_users);
204
205 acquired = bpf_kptr_xchg(&local->task, acquired);
206 if (acquired) {
207 err = 8;
208 bpf_obj_drop(local);
209 bpf_task_release(kptr);
210 bpf_task_release(acquired);
211 return 0;
212 }
213
214 bpf_obj_drop(local);
215
216 bpf_probe_read_kernel(&refcnt_after_drop, sizeof(refcnt_after_drop), &kptr->rcu_users);
217 if (refcnt != refcnt_after_drop + 1) {
218 err = 9;
219 bpf_task_release(kptr);
220 return 0;
221 }
222
223 bpf_task_release(kptr);
224 return 0;
225 }
226
227 SEC("tp_btf/task_newtask")
BPF_PROG(test_task_map_acquire_release,struct task_struct * task,u64 clone_flags)228 int BPF_PROG(test_task_map_acquire_release, struct task_struct *task, u64 clone_flags)
229 {
230 struct task_struct *kptr;
231 struct __tasks_kfunc_map_value *v;
232 long status;
233
234 if (!is_test_kfunc_task())
235 return 0;
236
237 status = tasks_kfunc_map_insert(task);
238 if (status) {
239 err = 1;
240 return 0;
241 }
242
243 v = tasks_kfunc_map_value_lookup(task);
244 if (!v) {
245 err = 2;
246 return 0;
247 }
248
249 bpf_rcu_read_lock();
250 kptr = v->task;
251 if (!kptr) {
252 err = 3;
253 } else {
254 kptr = bpf_task_acquire(kptr);
255 if (!kptr)
256 err = 4;
257 else
258 bpf_task_release(kptr);
259 }
260 bpf_rcu_read_unlock();
261
262 return 0;
263 }
264
265 SEC("tp_btf/task_newtask")
BPF_PROG(test_task_current_acquire_release,struct task_struct * task,u64 clone_flags)266 int BPF_PROG(test_task_current_acquire_release, struct task_struct *task, u64 clone_flags)
267 {
268 struct task_struct *current, *acquired;
269
270 if (!is_test_kfunc_task())
271 return 0;
272
273 current = bpf_get_current_task_btf();
274 acquired = bpf_task_acquire(current);
275 if (acquired)
276 bpf_task_release(acquired);
277 else
278 err = 1;
279
280 return 0;
281 }
282
lookup_compare_pid(const struct task_struct * p)283 static void lookup_compare_pid(const struct task_struct *p)
284 {
285 struct task_struct *acquired;
286
287 acquired = bpf_task_from_pid(p->pid);
288 if (!acquired) {
289 err = 1;
290 return;
291 }
292
293 if (acquired->pid != p->pid)
294 err = 2;
295 bpf_task_release(acquired);
296 }
297
298 SEC("tp_btf/task_newtask")
BPF_PROG(test_task_from_pid_arg,struct task_struct * task,u64 clone_flags)299 int BPF_PROG(test_task_from_pid_arg, struct task_struct *task, u64 clone_flags)
300 {
301 if (!is_test_kfunc_task())
302 return 0;
303
304 lookup_compare_pid(task);
305 return 0;
306 }
307
308 SEC("tp_btf/task_newtask")
BPF_PROG(test_task_from_pid_current,struct task_struct * task,u64 clone_flags)309 int BPF_PROG(test_task_from_pid_current, struct task_struct *task, u64 clone_flags)
310 {
311 if (!is_test_kfunc_task())
312 return 0;
313
314 lookup_compare_pid(bpf_get_current_task_btf());
315 return 0;
316 }
317
is_pid_lookup_valid(s32 pid)318 static int is_pid_lookup_valid(s32 pid)
319 {
320 struct task_struct *acquired;
321
322 acquired = bpf_task_from_pid(pid);
323 if (acquired) {
324 bpf_task_release(acquired);
325 return 1;
326 }
327
328 return 0;
329 }
330
331 SEC("tp_btf/task_newtask")
BPF_PROG(test_task_from_pid_invalid,struct task_struct * task,u64 clone_flags)332 int BPF_PROG(test_task_from_pid_invalid, struct task_struct *task, u64 clone_flags)
333 {
334 if (!is_test_kfunc_task())
335 return 0;
336
337 bpf_strncmp(task->comm, 12, "foo");
338 bpf_strncmp(task->comm, 16, "foo");
339 bpf_strncmp(&task->comm[8], 4, "foo");
340
341 if (is_pid_lookup_valid(-1)) {
342 err = 1;
343 return 0;
344 }
345
346 if (is_pid_lookup_valid(0xcafef00d)) {
347 err = 2;
348 return 0;
349 }
350
351 return 0;
352 }
353
354 SEC("tp_btf/task_newtask")
BPF_PROG(task_kfunc_acquire_trusted_walked,struct task_struct * task,u64 clone_flags)355 int BPF_PROG(task_kfunc_acquire_trusted_walked, struct task_struct *task, u64 clone_flags)
356 {
357 struct task_struct *acquired;
358
359 /* task->group_leader is listed as a trusted, non-NULL field of task struct. */
360 acquired = bpf_task_acquire(task->group_leader);
361 if (acquired)
362 bpf_task_release(acquired);
363 else
364 err = 1;
365
366
367 return 0;
368 }
369
370 SEC("fentry/" SYS_PREFIX "sys_getpgid")
BPF_PROG(task_kfunc_acquire_after_spin_unlock_non_sleepable)371 int BPF_PROG(task_kfunc_acquire_after_spin_unlock_non_sleepable)
372 {
373 struct task_kptr_lock_value *v;
374 struct task_struct *task, *acquired;
375 int key = 0;
376
377 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key);
378 if (!v)
379 return 0;
380
381 bpf_spin_lock(&v->lock);
382 task = v->task;
383 bpf_spin_unlock(&v->lock);
384 if (!task)
385 return 0;
386
387 acquired = bpf_task_acquire(task);
388 if (acquired)
389 bpf_task_release(acquired);
390 return 0;
391 }
392
393 SEC("fentry.s/" SYS_PREFIX "sys_getpgid")
BPF_PROG(task_kfunc_acquire_after_spin_unlock_explicit_rcu)394 int BPF_PROG(task_kfunc_acquire_after_spin_unlock_explicit_rcu)
395 {
396 struct task_kptr_lock_value *v;
397 struct task_struct *task, *acquired;
398 int key = 0;
399
400 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key);
401 if (!v)
402 return 0;
403
404 bpf_rcu_read_lock();
405 bpf_spin_lock(&v->lock);
406 task = v->task;
407 bpf_spin_unlock(&v->lock);
408 if (task) {
409 acquired = bpf_task_acquire(task);
410 if (acquired)
411 bpf_task_release(acquired);
412 }
413 bpf_rcu_read_unlock();
414 return 0;
415 }
416
417 SEC("fentry.s/" SYS_PREFIX "sys_getpgid")
BPF_PROG(task_kfunc_acquire_after_spin_unlock_preempt_disabled)418 int BPF_PROG(task_kfunc_acquire_after_spin_unlock_preempt_disabled)
419 {
420 struct task_kptr_lock_value *v;
421 struct task_struct *task, *acquired;
422 int key = 0;
423
424 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key);
425 if (!v)
426 return 0;
427
428 bpf_preempt_disable();
429 bpf_spin_lock(&v->lock);
430 task = v->task;
431 bpf_spin_unlock(&v->lock);
432 if (task) {
433 acquired = bpf_task_acquire(task);
434 if (acquired)
435 bpf_task_release(acquired);
436 }
437 bpf_preempt_enable();
438 return 0;
439 }
440
441 SEC("fentry.s/" SYS_PREFIX "sys_getpgid")
BPF_PROG(task_kfunc_acquire_after_spin_unlock_irq_disabled)442 int BPF_PROG(task_kfunc_acquire_after_spin_unlock_irq_disabled)
443 {
444 struct task_kptr_lock_value *v;
445 struct task_struct *task, *acquired;
446 unsigned long flags;
447 int key = 0;
448
449 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key);
450 if (!v)
451 return 0;
452
453 bpf_local_irq_save(&flags);
454 bpf_spin_lock(&v->lock);
455 task = v->task;
456 bpf_spin_unlock(&v->lock);
457 if (task) {
458 acquired = bpf_task_acquire(task);
459 if (acquired)
460 bpf_task_release(acquired);
461 }
462 bpf_local_irq_restore(&flags);
463 return 0;
464 }
465
466 SEC("fentry.s/" SYS_PREFIX "sys_getpgid")
BPF_PROG(task_kfunc_acquire_after_rcu_unlock_preempt_disabled)467 int BPF_PROG(task_kfunc_acquire_after_rcu_unlock_preempt_disabled)
468 {
469 struct task_kptr_lock_value *v;
470 struct task_struct *task, *acquired;
471 int key = 0;
472
473 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key);
474 if (!v)
475 return 0;
476
477 bpf_preempt_disable();
478 bpf_rcu_read_lock();
479 task = v->task;
480 bpf_rcu_read_unlock();
481 if (task) {
482 acquired = bpf_task_acquire(task);
483 if (acquired)
484 bpf_task_release(acquired);
485 }
486 bpf_preempt_enable();
487 return 0;
488 }
489
490 SEC("fentry.s/" SYS_PREFIX "sys_getpgid")
BPF_PROG(task_kfunc_acquire_after_rcu_unlock_irq_disabled)491 int BPF_PROG(task_kfunc_acquire_after_rcu_unlock_irq_disabled)
492 {
493 struct task_kptr_lock_value *v;
494 struct task_struct *task, *acquired;
495 unsigned long flags;
496 int key = 0;
497
498 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key);
499 if (!v)
500 return 0;
501
502 bpf_local_irq_save(&flags);
503 bpf_rcu_read_lock();
504 task = v->task;
505 bpf_rcu_read_unlock();
506 if (task) {
507 acquired = bpf_task_acquire(task);
508 if (acquired)
509 bpf_task_release(acquired);
510 }
511 bpf_local_irq_restore(&flags);
512 return 0;
513 }
514
515 SEC("fentry.s/" SYS_PREFIX "sys_getpgid")
BPF_PROG(task_kfunc_acquire_after_preempt_enable_explicit_rcu)516 int BPF_PROG(task_kfunc_acquire_after_preempt_enable_explicit_rcu)
517 {
518 struct task_kptr_lock_value *v;
519 struct task_struct *task, *acquired;
520 int key = 0;
521
522 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key);
523 if (!v)
524 return 0;
525
526 bpf_preempt_disable();
527 task = v->task;
528 bpf_rcu_read_lock();
529 bpf_preempt_enable();
530 if (task) {
531 acquired = bpf_task_acquire(task);
532 if (acquired)
533 bpf_task_release(acquired);
534 }
535 bpf_rcu_read_unlock();
536 return 0;
537 }
538
539 SEC("fentry.s/" SYS_PREFIX "sys_getpgid")
BPF_PROG(task_kfunc_acquire_after_irq_restore_explicit_rcu)540 int BPF_PROG(task_kfunc_acquire_after_irq_restore_explicit_rcu)
541 {
542 struct task_kptr_lock_value *v;
543 struct task_struct *task, *acquired;
544 unsigned long flags;
545 int key = 0;
546
547 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key);
548 if (!v)
549 return 0;
550
551 bpf_local_irq_save(&flags);
552 task = v->task;
553 bpf_rcu_read_lock();
554 bpf_local_irq_restore(&flags);
555 if (task) {
556 acquired = bpf_task_acquire(task);
557 if (acquired)
558 bpf_task_release(acquired);
559 }
560 bpf_rcu_read_unlock();
561 return 0;
562 }
563
564 SEC("syscall")
test_task_from_vpid_current(const void * ctx)565 int test_task_from_vpid_current(const void *ctx)
566 {
567 struct task_struct *current, *v_task;
568
569 v_task = bpf_task_from_vpid(1);
570 if (!v_task) {
571 err = 1;
572 return 0;
573 }
574
575 current = bpf_get_current_task_btf();
576
577 /* The current process should be the init process (pid 1) in the new pid namespace. */
578 if (current != v_task)
579 err = 2;
580
581 bpf_task_release(v_task);
582 return 0;
583 }
584
585 SEC("syscall")
test_task_from_vpid_invalid(const void * ctx)586 int test_task_from_vpid_invalid(const void *ctx)
587 {
588 struct task_struct *v_task;
589
590 v_task = bpf_task_from_vpid(-1);
591 if (v_task) {
592 err = 1;
593 goto err;
594 }
595
596 /* There should be only one process (current process) in the new pid namespace. */
597 v_task = bpf_task_from_vpid(2);
598 if (v_task) {
599 err = 2;
600 goto err;
601 }
602
603 v_task = bpf_task_from_vpid(9999);
604 if (v_task) {
605 err = 3;
606 goto err;
607 }
608
609 return 0;
610 err:
611 bpf_task_release(v_task);
612 return 0;
613 }
614