1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3
4 #include "vmlinux.h"
5 #include <bpf/bpf_helpers.h>
6 #include <bpf/bpf_tracing.h>
7 #include "bpf_tracing_net.h"
8 #include "bpf_misc.h"
9
10 /* clang considers 'sum += 1' as usage but 'sum++' as non-usage. GCC
11 * is more consistent and considers both 'sum += 1' and 'sum++' as
12 * non-usage. This triggers warnings in the functions below.
13 *
14 * Starting with GCC 16 -Wunused-but-set-variable=2 can be used to
15 * mimic clang's behavior. */
16 #if !defined(__clang__) && __GNUC__ > 15
17 #pragma GCC diagnostic ignored "-Wunused-but-set-variable"
18 #endif
19
20 char _license[] SEC("license") = "GPL";
21
22 struct {
23 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
24 __uint(map_flags, BPF_F_NO_PREALLOC);
25 __type(key, int);
26 __type(value, long);
27 } map_a SEC(".maps");
28
29 __u32 user_data, target_pid;
30 __s32 key_serial;
31 __u64 flags, task_storage_val, cgroup_id;
32
33 struct bpf_key *bpf_lookup_user_key(__s32 serial, __u64 flags) __ksym;
34 void bpf_key_put(struct bpf_key *key) __ksym;
35 void bpf_rcu_read_lock(void) __ksym;
36 void bpf_rcu_read_unlock(void) __ksym;
37 struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
38 void bpf_task_release(struct task_struct *p) __ksym;
39
40 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
get_cgroup_id(void * ctx)41 int get_cgroup_id(void *ctx)
42 {
43 struct task_struct *task;
44 struct css_set *cgroups;
45
46 task = bpf_get_current_task_btf();
47 if (task->pid != target_pid)
48 return 0;
49
50 /* simulate bpf_get_current_cgroup_id() helper */
51 bpf_rcu_read_lock();
52 cgroups = task->cgroups;
53 if (!cgroups)
54 goto unlock;
55 cgroup_id = cgroups->dfl_cgrp->kn->id;
56 unlock:
57 bpf_rcu_read_unlock();
58 return 0;
59 }
60
61 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
task_succ(void * ctx)62 int task_succ(void *ctx)
63 {
64 struct task_struct *task, *real_parent;
65 long init_val = 2;
66 long *ptr;
67
68 task = bpf_get_current_task_btf();
69 if (task->pid != target_pid)
70 return 0;
71
72 bpf_rcu_read_lock();
73 /* region including helper using rcu ptr real_parent */
74 real_parent = task->real_parent;
75 if (!real_parent)
76 goto out;
77 ptr = bpf_task_storage_get(&map_a, real_parent, &init_val,
78 BPF_LOCAL_STORAGE_GET_F_CREATE);
79 if (!ptr)
80 goto out;
81 ptr = bpf_task_storage_get(&map_a, real_parent, 0, 0);
82 if (!ptr)
83 goto out;
84 task_storage_val = *ptr;
85 out:
86 bpf_rcu_read_unlock();
87 return 0;
88 }
89
90 SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
no_lock(void * ctx)91 int no_lock(void *ctx)
92 {
93 struct task_struct *task, *real_parent;
94
95 /* old style ptr_to_btf_id is not allowed in sleepable */
96 task = bpf_get_current_task_btf();
97 real_parent = task->real_parent;
98 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
99 return 0;
100 }
101
102 SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
two_regions(void * ctx)103 int two_regions(void *ctx)
104 {
105 struct task_struct *task, *real_parent;
106
107 /* two regions */
108 task = bpf_get_current_task_btf();
109 bpf_rcu_read_lock();
110 bpf_rcu_read_unlock();
111 bpf_rcu_read_lock();
112 real_parent = task->real_parent;
113 if (!real_parent)
114 goto out;
115 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
116 out:
117 bpf_rcu_read_unlock();
118 return 0;
119 }
120
121 SEC("?fentry/" SYS_PREFIX "sys_getpgid")
non_sleepable_1(void * ctx)122 int non_sleepable_1(void *ctx)
123 {
124 struct task_struct *task, *real_parent;
125
126 task = bpf_get_current_task_btf();
127 bpf_rcu_read_lock();
128 real_parent = task->real_parent;
129 if (!real_parent)
130 goto out;
131 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
132 out:
133 bpf_rcu_read_unlock();
134 return 0;
135 }
136
137 SEC("?fentry/" SYS_PREFIX "sys_getpgid")
non_sleepable_2(void * ctx)138 int non_sleepable_2(void *ctx)
139 {
140 struct task_struct *task, *real_parent;
141
142 bpf_rcu_read_lock();
143 task = bpf_get_current_task_btf();
144 bpf_rcu_read_unlock();
145
146 bpf_rcu_read_lock();
147 real_parent = task->real_parent;
148 if (!real_parent)
149 goto out;
150 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
151 out:
152 bpf_rcu_read_unlock();
153 return 0;
154 }
155
156 SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
task_acquire(void * ctx)157 int task_acquire(void *ctx)
158 {
159 struct task_struct *task, *real_parent, *gparent;
160
161 task = bpf_get_current_task_btf();
162 bpf_rcu_read_lock();
163 real_parent = task->real_parent;
164 if (!real_parent)
165 goto out;
166
167 /* rcu_ptr->rcu_field */
168 gparent = real_parent->real_parent;
169 if (!gparent)
170 goto out;
171
172 /* acquire a reference which can be used outside rcu read lock region */
173 gparent = bpf_task_acquire(gparent);
174 if (!gparent)
175 goto out;
176
177 (void)bpf_task_storage_get(&map_a, gparent, 0, 0);
178 bpf_task_release(gparent);
179 out:
180 bpf_rcu_read_unlock();
181 return 0;
182 }
183
184 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
miss_lock(void * ctx)185 int miss_lock(void *ctx)
186 {
187 struct task_struct *task;
188
189 /* missing bpf_rcu_read_lock() */
190 task = bpf_get_current_task_btf();
191 bpf_rcu_read_lock();
192 (void)bpf_task_storage_get(&map_a, task, 0, 0);
193 bpf_rcu_read_unlock();
194 bpf_rcu_read_unlock();
195 return 0;
196 }
197
198 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
miss_unlock(void * ctx)199 int miss_unlock(void *ctx)
200 {
201 struct task_struct *task;
202
203 /* missing bpf_rcu_read_unlock() */
204 task = bpf_get_current_task_btf();
205 bpf_rcu_read_lock();
206 (void)bpf_task_storage_get(&map_a, task, 0, 0);
207 return 0;
208 }
209
210 SEC("?fentry/" SYS_PREFIX "sys_getpgid")
non_sleepable_rcu_mismatch(void * ctx)211 int non_sleepable_rcu_mismatch(void *ctx)
212 {
213 struct task_struct *task, *real_parent;
214
215 task = bpf_get_current_task_btf();
216 /* non-sleepable: missing bpf_rcu_read_unlock() in one path */
217 bpf_rcu_read_lock();
218 real_parent = task->real_parent;
219 if (!real_parent)
220 goto out;
221 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
222 if (real_parent)
223 bpf_rcu_read_unlock();
224 out:
225 return 0;
226 }
227
228 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
inproper_sleepable_helper(void * ctx)229 int inproper_sleepable_helper(void *ctx)
230 {
231 struct task_struct *task, *real_parent;
232 struct pt_regs *regs;
233 __u32 value = 0;
234 void *ptr;
235
236 task = bpf_get_current_task_btf();
237 /* sleepable helper in rcu read lock region */
238 bpf_rcu_read_lock();
239 real_parent = task->real_parent;
240 if (!real_parent)
241 goto out;
242 regs = (struct pt_regs *)bpf_task_pt_regs(real_parent);
243 if (!regs)
244 goto out;
245
246 ptr = (void *)PT_REGS_IP(regs);
247 (void)bpf_copy_from_user_task(&value, sizeof(uint32_t), ptr, task, 0);
248 user_data = value;
249 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
250 out:
251 bpf_rcu_read_unlock();
252 return 0;
253 }
254
255 SEC("?lsm.s/bpf")
BPF_PROG(inproper_sleepable_kfunc,int cmd,union bpf_attr * attr,unsigned int size,bool kernel)256 int BPF_PROG(inproper_sleepable_kfunc, int cmd, union bpf_attr *attr, unsigned int size,
257 bool kernel)
258 {
259 struct bpf_key *bkey;
260
261 /* sleepable kfunc in rcu read lock region */
262 bpf_rcu_read_lock();
263 bkey = bpf_lookup_user_key(key_serial, flags);
264 bpf_rcu_read_unlock();
265 if (!bkey)
266 return -1;
267 bpf_key_put(bkey);
268
269 return 0;
270 }
271
272 SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
nested_rcu_region(void * ctx)273 int nested_rcu_region(void *ctx)
274 {
275 struct task_struct *task, *real_parent;
276
277 /* nested rcu read lock regions */
278 task = bpf_get_current_task_btf();
279 bpf_rcu_read_lock();
280 bpf_rcu_read_lock();
281 real_parent = task->real_parent;
282 if (!real_parent)
283 goto out;
284 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
285 out:
286 bpf_rcu_read_unlock();
287 bpf_rcu_read_unlock();
288 return 0;
289 }
290
291 SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
nested_rcu_region_unbalanced_1(void * ctx)292 int nested_rcu_region_unbalanced_1(void *ctx)
293 {
294 struct task_struct *task, *real_parent;
295
296 /* nested rcu read lock regions */
297 task = bpf_get_current_task_btf();
298 bpf_rcu_read_lock();
299 bpf_rcu_read_lock();
300 real_parent = task->real_parent;
301 if (!real_parent)
302 goto out;
303 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
304 out:
305 bpf_rcu_read_unlock();
306 bpf_rcu_read_unlock();
307 bpf_rcu_read_unlock();
308 return 0;
309 }
310
311 SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
nested_rcu_region_unbalanced_2(void * ctx)312 int nested_rcu_region_unbalanced_2(void *ctx)
313 {
314 struct task_struct *task, *real_parent;
315
316 /* nested rcu read lock regions */
317 task = bpf_get_current_task_btf();
318 bpf_rcu_read_lock();
319 bpf_rcu_read_lock();
320 bpf_rcu_read_lock();
321 real_parent = task->real_parent;
322 if (!real_parent)
323 goto out;
324 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
325 out:
326 bpf_rcu_read_unlock();
327 bpf_rcu_read_unlock();
328 return 0;
329 }
330
331 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
task_trusted_non_rcuptr(void * ctx)332 int task_trusted_non_rcuptr(void *ctx)
333 {
334 struct task_struct *task, *group_leader;
335
336 task = bpf_get_current_task_btf();
337 bpf_rcu_read_lock();
338 /* the pointer group_leader is explicitly marked as trusted */
339 group_leader = task->real_parent->group_leader;
340 (void)bpf_task_storage_get(&map_a, group_leader, 0, 0);
341 bpf_rcu_read_unlock();
342 return 0;
343 }
344
345 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
task_untrusted_rcuptr(void * ctx)346 int task_untrusted_rcuptr(void *ctx)
347 {
348 struct task_struct *task, *real_parent;
349
350 task = bpf_get_current_task_btf();
351 bpf_rcu_read_lock();
352 real_parent = task->real_parent;
353 bpf_rcu_read_unlock();
354 /* helper use of rcu ptr outside the rcu read lock region */
355 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
356 return 0;
357 }
358
359 SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
cross_rcu_region(void * ctx)360 int cross_rcu_region(void *ctx)
361 {
362 struct task_struct *task, *real_parent;
363
364 /* rcu ptr define/use in different regions */
365 task = bpf_get_current_task_btf();
366 bpf_rcu_read_lock();
367 real_parent = task->real_parent;
368 bpf_rcu_read_unlock();
369 bpf_rcu_read_lock();
370 (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
371 bpf_rcu_read_unlock();
372 return 0;
373 }
374
375 __noinline
static_subprog(void * ctx)376 static int static_subprog(void *ctx)
377 {
378 volatile int ret = 0;
379
380 if (bpf_get_prandom_u32())
381 return ret + 42;
382 return ret + bpf_get_prandom_u32();
383 }
384
385 __noinline
global_subprog(u64 a)386 int global_subprog(u64 a)
387 {
388 volatile int ret = a;
389
390 return ret + static_subprog(NULL);
391 }
392
393 __noinline
static_subprog_lock(void * ctx)394 static int static_subprog_lock(void *ctx)
395 {
396 volatile int ret = 0;
397
398 bpf_rcu_read_lock();
399 if (bpf_get_prandom_u32())
400 return ret + 42;
401 return ret + bpf_get_prandom_u32();
402 }
403
404 __noinline
global_subprog_lock(u64 a)405 int global_subprog_lock(u64 a)
406 {
407 volatile int ret = a;
408
409 return ret + static_subprog_lock(NULL);
410 }
411
412 __noinline
static_subprog_unlock(void * ctx)413 static int static_subprog_unlock(void *ctx)
414 {
415 volatile int ret = 0;
416
417 bpf_rcu_read_unlock();
418 if (bpf_get_prandom_u32())
419 return ret + 42;
420 return ret + bpf_get_prandom_u32();
421 }
422
423 __noinline
global_subprog_unlock(u64 a)424 int global_subprog_unlock(u64 a)
425 {
426 volatile int ret = a;
427
428 return ret + static_subprog_unlock(NULL);
429 }
430
431 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
rcu_read_lock_subprog(void * ctx)432 int rcu_read_lock_subprog(void *ctx)
433 {
434 volatile int ret = 0;
435
436 bpf_rcu_read_lock();
437 if (bpf_get_prandom_u32())
438 ret += static_subprog(ctx);
439 bpf_rcu_read_unlock();
440 return 0;
441 }
442
443 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
rcu_read_lock_global_subprog(void * ctx)444 int rcu_read_lock_global_subprog(void *ctx)
445 {
446 volatile int ret = 0;
447
448 bpf_rcu_read_lock();
449 if (bpf_get_prandom_u32())
450 ret += global_subprog(ret);
451 bpf_rcu_read_unlock();
452 return 0;
453 }
454
455 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
rcu_read_lock_subprog_lock(void * ctx)456 int rcu_read_lock_subprog_lock(void *ctx)
457 {
458 volatile int ret = 0;
459
460 ret += static_subprog_lock(ctx);
461 bpf_rcu_read_unlock();
462 return 0;
463 }
464
465 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
rcu_read_lock_global_subprog_lock(void * ctx)466 int rcu_read_lock_global_subprog_lock(void *ctx)
467 {
468 volatile int ret = 0;
469
470 ret += global_subprog_lock(ret);
471 bpf_rcu_read_unlock();
472 return 0;
473 }
474
475 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
rcu_read_lock_subprog_unlock(void * ctx)476 int rcu_read_lock_subprog_unlock(void *ctx)
477 {
478 volatile int ret = 0;
479
480 bpf_rcu_read_lock();
481 ret += static_subprog_unlock(ctx);
482 return 0;
483 }
484
485 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
rcu_read_lock_global_subprog_unlock(void * ctx)486 int rcu_read_lock_global_subprog_unlock(void *ctx)
487 {
488 volatile int ret = 0;
489
490 bpf_rcu_read_lock();
491 ret += global_subprog_unlock(ret);
492 return 0;
493 }
494
495 int __noinline
global_sleepable_helper_subprog(int i)496 global_sleepable_helper_subprog(int i)
497 {
498 if (i)
499 bpf_copy_from_user(&i, sizeof(i), NULL);
500 return i;
501 }
502
503 int __noinline
global_sleepable_kfunc_subprog(int i)504 global_sleepable_kfunc_subprog(int i)
505 {
506 if (i)
507 bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
508 global_subprog(i);
509 return i;
510 }
511
512 int __noinline
global_subprog_calling_sleepable_global(int i)513 global_subprog_calling_sleepable_global(int i)
514 {
515 if (!i)
516 global_sleepable_kfunc_subprog(i);
517 return i;
518 }
519
520 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
rcu_read_lock_sleepable_helper_global_subprog(void * ctx)521 int rcu_read_lock_sleepable_helper_global_subprog(void *ctx)
522 {
523 volatile int ret = 0;
524
525 bpf_rcu_read_lock();
526 ret += global_sleepable_helper_subprog(ret);
527 bpf_rcu_read_unlock();
528 return 0;
529 }
530
531 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
rcu_read_lock_sleepable_kfunc_global_subprog(void * ctx)532 int rcu_read_lock_sleepable_kfunc_global_subprog(void *ctx)
533 {
534 volatile int ret = 0;
535
536 bpf_rcu_read_lock();
537 ret += global_sleepable_kfunc_subprog(ret);
538 bpf_rcu_read_unlock();
539 return 0;
540 }
541
542 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
rcu_read_lock_sleepable_global_subprog_indirect(void * ctx)543 int rcu_read_lock_sleepable_global_subprog_indirect(void *ctx)
544 {
545 volatile int ret = 0;
546
547 bpf_rcu_read_lock();
548 ret += global_subprog_calling_sleepable_global(ret);
549 bpf_rcu_read_unlock();
550 return 0;
551 }
552
553 struct rcu_node_data {
554 long key;
555 struct bpf_rb_node node;
556 };
557
558 struct rcu_node_stash {
559 struct rcu_node_data __kptr *node;
560 };
561
562 /*
563 * Necessary so that LLVM emits BTF for rcu_node_data rather than just a
564 * fwd reference to it, same as in progs/local_kptr_stash.c.
565 */
566 struct rcu_node_data *just_here_because_btf_bug;
567
568 struct {
569 __uint(type, BPF_MAP_TYPE_ARRAY);
570 __uint(max_entries, 1);
571 __type(key, int);
572 __type(value, struct rcu_node_stash);
573 } node_stash SEC(".maps");
574
575 long non_own_ref_key;
576
577 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
non_own_ref_untrusted_ld(void * ctx)578 int non_own_ref_untrusted_ld(void *ctx)
579 {
580 struct rcu_node_stash *stash;
581 struct rcu_node_data *node;
582 int key = 0;
583
584 stash = bpf_map_lookup_elem(&node_stash, &key);
585 if (!stash)
586 return 0;
587 bpf_rcu_read_lock();
588 node = stash->node;
589 if (!node) {
590 bpf_rcu_read_unlock();
591 return 0;
592 }
593 bpf_rcu_read_unlock();
594 /*
595 * The unlock leaves node as PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED
596 * | NON_OWN_REF, and the load below has to get the BPF_PROBE_MEM
597 * rewrite for it, otherwise a bad address panics the kernel.
598 */
599 non_own_ref_key = node->key;
600 return 0;
601 }
602
603 long rcu_untrusted_wq_flags;
604
605 SEC("?tp_btf/tcp_probe")
BPF_PROG(rcu_untrusted_union_ld,struct sock * sk)606 int BPF_PROG(rcu_untrusted_union_ld, struct sock *sk)
607 {
608 struct socket_wq *wq;
609
610 /*
611 * sk_wq sits in a two member union, so btf_struct_walk() marks the
612 * pointer PTR_UNTRUSTED, and the __rcu tag on the member adds MEM_RCU
613 * on top of it. struct sock is not on the __safe_rcu_or_null allow
614 * list, hence the two stay combined and the load below has to get the
615 * BPF_PROBE_MEM rewrite for PTR_TO_BTF_ID | PTR_UNTRUSTED | MEM_RCU,
616 * otherwise a bad address panics the kernel.
617 *
618 * The __rcu tag only reaches BTF on a clang built kernel, that is, one
619 * with CONFIG_PAHOLE_HAS_BTF_TAG. On a gcc built kernel the walk yields
620 * a plain untrusted pointer, which is rewritten either way.
621 */
622 wq = sk->sk_wq;
623 if (!wq)
624 return 0;
625 rcu_untrusted_wq_flags = wq->flags;
626 return 0;
627 }
628