1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Augment the raw_syscalls tracepoints with the contents of the pointer arguments.
4 *
5 * This exactly matches what is marshalled into the raw_syscall:sys_enter
6 * payload expected by the 'perf trace' beautifiers.
7 */
8
9 #include "vmlinux.h"
10 #include "../trace_augment.h"
11
12 #include <bpf/bpf_helpers.h>
13 #include <linux/limits.h>
14
15 #define PERF_ALIGN(x, a) __PERF_ALIGN_MASK(x, (typeof(x))(a)-1)
16 #define __PERF_ALIGN_MASK(x, mask) (((x)+(mask))&~(mask))
17
18 /**
19 * is_power_of_2() - check if a value is a power of two
20 * @n: the value to check
21 *
22 * Determine whether some value is a power of two, where zero is *not*
23 * considered a power of two. Return: true if @n is a power of 2, otherwise
24 * false.
25 */
26 #define is_power_of_2(n) (n != 0 && ((n & (n - 1)) == 0))
27
28 #define MAX_CPUS 4096
29
30 /* bpf-output associated map */
31 struct __augmented_syscalls__ {
32 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
33 __type(key, int);
34 __type(value, __u32);
35 __uint(max_entries, MAX_CPUS);
36 } __augmented_syscalls__ SEC(".maps");
37
38 /*
39 * What to augment at entry?
40 *
41 * Pointer arg payloads (filenames, etc) passed from userspace to the kernel
42 */
43 struct syscalls_sys_enter {
44 __uint(type, BPF_MAP_TYPE_PROG_ARRAY);
45 __type(key, __u32);
46 __type(value, __u32);
47 __uint(max_entries, 512);
48 } syscalls_sys_enter SEC(".maps");
49
50 /*
51 * What to augment at exit?
52 *
53 * Pointer arg payloads returned from the kernel (struct stat, etc) to userspace.
54 */
55 struct syscalls_sys_exit {
56 __uint(type, BPF_MAP_TYPE_PROG_ARRAY);
57 __type(key, __u32);
58 __type(value, __u32);
59 __uint(max_entries, 512);
60 } syscalls_sys_exit SEC(".maps");
61
62 struct syscall_enter_args {
63 unsigned long long common_tp_fields;
64 long syscall_nr;
65 unsigned long args[6];
66 };
67
68 struct syscall_exit_args {
69 unsigned long long common_tp_fields;
70 long syscall_nr;
71 long ret;
72 };
73
74 /*
75 * Desired design of maximum size and alignment (see RFC2553)
76 */
77 #define SS_MAXSIZE 128 /* Implementation specific max size */
78
79 typedef unsigned short sa_family_t;
80
81 /*
82 * FIXME: Should come from system headers
83 *
84 * The definition uses anonymous union and struct in order to control the
85 * default alignment.
86 */
87 struct sockaddr_storage {
88 union {
89 struct {
90 sa_family_t ss_family; /* address family */
91 /* Following field(s) are implementation specific */
92 char __data[SS_MAXSIZE - sizeof(unsigned short)];
93 /* space to achieve desired size, */
94 /* _SS_MAXSIZE value minus size of ss_family */
95 };
96 void *__align; /* implementation specific desired alignment */
97 };
98 };
99
100 struct augmented_arg {
101 unsigned int size;
102 int err;
103 union {
104 char value[PATH_MAX];
105 struct sockaddr_storage saddr;
106 };
107 };
108
109 struct pids_filtered {
110 __uint(type, BPF_MAP_TYPE_HASH);
111 __type(key, pid_t);
112 __type(value, bool);
113 __uint(max_entries, 64);
114 } pids_filtered SEC(".maps");
115
116 struct augmented_args_payload {
117 struct syscall_enter_args args;
118 struct augmented_arg arg, arg2; // We have to reserve space for two arguments (rename, etc)
119 };
120
121 // We need more tmp space than the BPF stack can give us
122 struct augmented_args_tmp {
123 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
124 __type(key, int);
125 __type(value, struct augmented_args_payload);
126 __uint(max_entries, 1);
127 } augmented_args_tmp SEC(".maps");
128
129 struct beauty_map_enter {
130 __uint(type, BPF_MAP_TYPE_HASH);
131 __type(key, int);
132 __type(value, __u32[6]);
133 __uint(max_entries, 512);
134 } beauty_map_enter SEC(".maps");
135
136 struct beauty_payload_enter {
137 struct syscall_enter_args args;
138 struct augmented_arg aug_args[6];
139 };
140
141 struct beauty_payload_enter_map {
142 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
143 __type(key, int);
144 __type(value, struct beauty_payload_enter);
145 __uint(max_entries, 1);
146 } beauty_payload_enter_map SEC(".maps");
147
augmented_args_payload(void)148 static inline struct augmented_args_payload *augmented_args_payload(void)
149 {
150 int key = 0;
151 return bpf_map_lookup_elem(&augmented_args_tmp, &key);
152 }
153
augmented__output(void * ctx,struct augmented_args_payload * args,int len)154 static inline int augmented__output(void *ctx, struct augmented_args_payload *args, int len)
155 {
156 /* If perf_event_output fails, return non-zero so that it gets recorded unaugmented */
157 return bpf_perf_event_output(ctx, &__augmented_syscalls__, BPF_F_CURRENT_CPU, args, len);
158 }
159
augmented__beauty_output(void * ctx,void * data,int len)160 static inline int augmented__beauty_output(void *ctx, void *data, int len)
161 {
162 return bpf_perf_event_output(ctx, &__augmented_syscalls__, BPF_F_CURRENT_CPU, data, len);
163 }
164
165 static inline
augmented_arg__read_str(struct augmented_arg * augmented_arg,const void * arg,unsigned int arg_len)166 unsigned int augmented_arg__read_str(struct augmented_arg *augmented_arg, const void *arg, unsigned int arg_len)
167 {
168 unsigned int augmented_len = sizeof(*augmented_arg);
169 int string_len = bpf_probe_read_user_str(&augmented_arg->value, arg_len, arg);
170
171 augmented_arg->size = augmented_arg->err = 0;
172 /*
173 * probe_read_str may return < 0, e.g. -EFAULT
174 * So we leave that in the augmented_arg->size that userspace will
175 */
176 if (string_len > 0) {
177 augmented_len -= sizeof(augmented_arg->value) - string_len;
178 _Static_assert(is_power_of_2(sizeof(augmented_arg->value)), "sizeof(augmented_arg->value) needs to be a power of two");
179 augmented_len &= sizeof(augmented_arg->value) - 1;
180 augmented_arg->size = string_len;
181 } else {
182 /*
183 * So that username notice the error while still being able
184 * to skip this augmented arg record
185 */
186 augmented_arg->err = string_len;
187 augmented_len = offsetof(struct augmented_arg, value);
188 }
189
190 return augmented_len;
191 }
192
193 SEC("tp/raw_syscalls/sys_enter")
syscall_unaugmented(struct syscall_enter_args * args)194 int syscall_unaugmented(struct syscall_enter_args *args)
195 {
196 return 1;
197 }
198
199 /*
200 * These will be tail_called from SEC("raw_syscalls:sys_enter"), so will find in
201 * augmented_args_tmp what was read by that raw_syscalls:sys_enter and go
202 * on from there, reading the first syscall arg as a string, i.e. open's
203 * filename.
204 */
205 SEC("tp/syscalls/sys_enter_connect")
sys_enter_connect(struct syscall_enter_args * args)206 int sys_enter_connect(struct syscall_enter_args *args)
207 {
208 struct augmented_args_payload *augmented_args = augmented_args_payload();
209 const void *sockaddr_arg = (const void *)args->args[1];
210 unsigned int socklen = args->args[2];
211 unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs
212
213 if (augmented_args == NULL)
214 return 1; /* Failure: don't filter */
215
216 _Static_assert(is_power_of_2(sizeof(augmented_args->arg.saddr)), "sizeof(augmented_args->arg.saddr) needs to be a power of two");
217 socklen &= sizeof(augmented_args->arg.saddr) - 1;
218
219 bpf_probe_read_user(&augmented_args->arg.saddr, socklen, sockaddr_arg);
220 augmented_args->arg.size = socklen;
221 augmented_args->arg.err = 0;
222
223 return augmented__output(args, augmented_args, len + socklen);
224 }
225
226 SEC("tp/syscalls/sys_enter_sendto")
sys_enter_sendto(struct syscall_enter_args * args)227 int sys_enter_sendto(struct syscall_enter_args *args)
228 {
229 struct augmented_args_payload *augmented_args = augmented_args_payload();
230 const void *sockaddr_arg = (const void *)args->args[4];
231 unsigned int socklen = args->args[5];
232 unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs
233
234 if (augmented_args == NULL)
235 return 1; /* Failure: don't filter */
236
237 socklen &= sizeof(augmented_args->arg.saddr) - 1;
238
239 bpf_probe_read_user(&augmented_args->arg.saddr, socklen, sockaddr_arg);
240
241 return augmented__output(args, augmented_args, len + socklen);
242 }
243
244 SEC("tp/syscalls/sys_enter_open")
sys_enter_open(struct syscall_enter_args * args)245 int sys_enter_open(struct syscall_enter_args *args)
246 {
247 struct augmented_args_payload *augmented_args = augmented_args_payload();
248 const void *filename_arg = (const void *)args->args[0];
249 unsigned int len = sizeof(augmented_args->args);
250
251 if (augmented_args == NULL)
252 return 1; /* Failure: don't filter */
253
254 len += augmented_arg__read_str(&augmented_args->arg, filename_arg, sizeof(augmented_args->arg.value));
255
256 return augmented__output(args, augmented_args, len);
257 }
258
259 SEC("tp/syscalls/sys_enter_openat")
sys_enter_openat(struct syscall_enter_args * args)260 int sys_enter_openat(struct syscall_enter_args *args)
261 {
262 struct augmented_args_payload *augmented_args = augmented_args_payload();
263 const void *filename_arg = (const void *)args->args[1];
264 unsigned int len = sizeof(augmented_args->args);
265
266 if (augmented_args == NULL)
267 return 1; /* Failure: don't filter */
268
269 len += augmented_arg__read_str(&augmented_args->arg, filename_arg, sizeof(augmented_args->arg.value));
270
271 return augmented__output(args, augmented_args, len);
272 }
273
274 SEC("tp/syscalls/sys_enter_rename")
sys_enter_rename(struct syscall_enter_args * args)275 int sys_enter_rename(struct syscall_enter_args *args)
276 {
277 struct augmented_args_payload *augmented_args = augmented_args_payload();
278 const void *oldpath_arg = (const void *)args->args[0],
279 *newpath_arg = (const void *)args->args[1];
280 unsigned int len = sizeof(augmented_args->args), oldpath_len, newpath_len;
281
282 if (augmented_args == NULL)
283 return 1; /* Failure: don't filter */
284
285 len += 2 * sizeof(u64); // The overhead of size and err, just before the payload...
286
287 oldpath_len = augmented_arg__read_str(&augmented_args->arg, oldpath_arg, sizeof(augmented_args->arg.value));
288 augmented_args->arg.size = PERF_ALIGN(oldpath_len + 1, sizeof(u64));
289 len += augmented_args->arg.size;
290
291 /* Every read from userspace is limited to value size */
292 if (augmented_args->arg.size > sizeof(augmented_args->arg.value))
293 return 1; /* Failure: don't filter */
294
295 struct augmented_arg *arg2 = (void *)&augmented_args->arg.value + augmented_args->arg.size;
296
297 newpath_len = augmented_arg__read_str(arg2, newpath_arg, sizeof(augmented_args->arg.value));
298 arg2->size = newpath_len;
299
300 len += newpath_len;
301
302 return augmented__output(args, augmented_args, len);
303 }
304
305 SEC("tp/syscalls/sys_enter_renameat2")
sys_enter_renameat2(struct syscall_enter_args * args)306 int sys_enter_renameat2(struct syscall_enter_args *args)
307 {
308 struct augmented_args_payload *augmented_args = augmented_args_payload();
309 const void *oldpath_arg = (const void *)args->args[1],
310 *newpath_arg = (const void *)args->args[3];
311 unsigned int len = sizeof(augmented_args->args), oldpath_len, newpath_len;
312
313 if (augmented_args == NULL)
314 return 1; /* Failure: don't filter */
315
316 len += 2 * sizeof(u64); // The overhead of size and err, just before the payload...
317
318 oldpath_len = augmented_arg__read_str(&augmented_args->arg, oldpath_arg, sizeof(augmented_args->arg.value));
319 augmented_args->arg.size = PERF_ALIGN(oldpath_len + 1, sizeof(u64));
320 len += augmented_args->arg.size;
321
322 /* Every read from userspace is limited to value size */
323 if (augmented_args->arg.size > sizeof(augmented_args->arg.value))
324 return 1; /* Failure: don't filter */
325
326 struct augmented_arg *arg2 = (void *)&augmented_args->arg.value + augmented_args->arg.size;
327
328 newpath_len = augmented_arg__read_str(arg2, newpath_arg, sizeof(augmented_args->arg.value));
329 arg2->size = newpath_len;
330
331 len += newpath_len;
332
333 return augmented__output(args, augmented_args, len);
334 }
335
336 #define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */
337
338 // we need just the start, get the size to then copy it
339 struct perf_event_attr_size {
340 __u32 type;
341 /*
342 * Size of the attr structure, for fwd/bwd compat.
343 */
344 __u32 size;
345 };
346
347 SEC("tp/syscalls/sys_enter_perf_event_open")
sys_enter_perf_event_open(struct syscall_enter_args * args)348 int sys_enter_perf_event_open(struct syscall_enter_args *args)
349 {
350 struct augmented_args_payload *augmented_args = augmented_args_payload();
351 const struct perf_event_attr_size *attr = (const struct perf_event_attr_size *)args->args[0], *attr_read;
352 unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs
353
354 if (augmented_args == NULL)
355 goto failure;
356
357 if (bpf_probe_read_user(&augmented_args->arg.value, sizeof(*attr), attr) < 0)
358 goto failure;
359
360 attr_read = (const struct perf_event_attr_size *)augmented_args->arg.value;
361
362 __u32 size = attr_read->size;
363
364 if (!size)
365 size = PERF_ATTR_SIZE_VER0;
366
367 if (size > sizeof(augmented_args->arg.value))
368 goto failure;
369
370 // Now that we read attr->size and tested it against the size limits, read it completely
371 if (bpf_probe_read_user(&augmented_args->arg.value, size, attr) < 0)
372 goto failure;
373
374 return augmented__output(args, augmented_args, len + size);
375 failure:
376 return 1; /* Failure: don't filter */
377 }
378
379 SEC("tp/syscalls/sys_enter_clock_nanosleep")
sys_enter_clock_nanosleep(struct syscall_enter_args * args)380 int sys_enter_clock_nanosleep(struct syscall_enter_args *args)
381 {
382 struct augmented_args_payload *augmented_args = augmented_args_payload();
383 const void *rqtp_arg = (const void *)args->args[2];
384 unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs
385 __u32 size = sizeof(struct timespec64);
386
387 if (augmented_args == NULL)
388 goto failure;
389
390 if (size > sizeof(augmented_args->arg.value))
391 goto failure;
392
393 bpf_probe_read_user(&augmented_args->arg.value, size, rqtp_arg);
394
395 return augmented__output(args, augmented_args, len + size);
396 failure:
397 return 1; /* Failure: don't filter */
398 }
399
400 SEC("tp/syscalls/sys_enter_nanosleep")
sys_enter_nanosleep(struct syscall_enter_args * args)401 int sys_enter_nanosleep(struct syscall_enter_args *args)
402 {
403 struct augmented_args_payload *augmented_args = augmented_args_payload();
404 const void *req_arg = (const void *)args->args[0];
405 unsigned int len = sizeof(augmented_args->args);
406 __u32 size = sizeof(struct timespec64);
407
408 if (augmented_args == NULL)
409 goto failure;
410
411 if (size > sizeof(augmented_args->arg.value))
412 goto failure;
413
414 bpf_probe_read_user(&augmented_args->arg.value, size, req_arg);
415
416 return augmented__output(args, augmented_args, len + size);
417 failure:
418 return 1; /* Failure: don't filter */
419 }
420
getpid(void)421 static pid_t getpid(void)
422 {
423 return bpf_get_current_pid_tgid();
424 }
425
pid_filter__has(struct pids_filtered * pids,pid_t pid)426 static bool pid_filter__has(struct pids_filtered *pids, pid_t pid)
427 {
428 return bpf_map_lookup_elem(pids, &pid) != NULL;
429 }
430
augment_sys_enter(void * ctx,struct syscall_enter_args * args)431 static int augment_sys_enter(void *ctx, struct syscall_enter_args *args)
432 {
433 bool augmented, do_output = false;
434 int zero = 0, size, aug_size, index,
435 value_size = sizeof(struct augmented_arg) - offsetof(struct augmented_arg, value);
436 u64 output = 0; /* has to be u64, otherwise it won't pass the verifier */
437 unsigned int nr, *beauty_map;
438 struct beauty_payload_enter *payload;
439 void *arg, *payload_offset;
440
441 /* fall back to do predefined tail call */
442 if (args == NULL)
443 return 1;
444
445 /* use syscall number to get beauty_map entry */
446 nr = (__u32)args->syscall_nr;
447 beauty_map = bpf_map_lookup_elem(&beauty_map_enter, &nr);
448
449 /* set up payload for output */
450 payload = bpf_map_lookup_elem(&beauty_payload_enter_map, &zero);
451 payload_offset = (void *)&payload->aug_args;
452
453 if (beauty_map == NULL || payload == NULL)
454 return 1;
455
456 /* copy the sys_enter header, which has the syscall_nr */
457 __builtin_memcpy(&payload->args, args, sizeof(struct syscall_enter_args));
458
459 /*
460 * Determine what type of argument and how many bytes to read from user space, using the
461 * value in the beauty_map. This is the relation of parameter type and its corresponding
462 * value in the beauty map, and how many bytes we read eventually:
463 *
464 * string: 1 -> size of string
465 * struct: size of struct -> size of struct
466 * buffer: -1 * (index of paired len) -> value of paired len (maximum: TRACE_AUG_MAX_BUF)
467 */
468 for (int i = 0; i < 6; i++) {
469 arg = (void *)args->args[i];
470 augmented = false;
471 size = beauty_map[i];
472 aug_size = size; /* size of the augmented data read from user space */
473
474 if (size == 0 || arg == NULL)
475 continue;
476
477 if (size == 1) { /* string */
478 aug_size = bpf_probe_read_user_str(((struct augmented_arg *)payload_offset)->value, value_size, arg);
479 /* minimum of 0 to pass the verifier */
480 if (aug_size < 0)
481 aug_size = 0;
482
483 augmented = true;
484 } else if (size > 0 && size <= value_size) { /* struct */
485 if (!bpf_probe_read_user(((struct augmented_arg *)payload_offset)->value, size, arg))
486 augmented = true;
487 } else if (size < 0 && size >= -6) { /* buffer */
488 index = -(size + 1);
489 barrier_var(index); // Prevent clang (noticed with v18) from removing the &= 7 trick.
490 index &= 7; // Satisfy the bounds checking with the verifier in some kernels.
491 aug_size = args->args[index];
492
493 if (aug_size > TRACE_AUG_MAX_BUF)
494 aug_size = TRACE_AUG_MAX_BUF;
495
496 if (aug_size > 0) {
497 if (!bpf_probe_read_user(((struct augmented_arg *)payload_offset)->value, aug_size, arg))
498 augmented = true;
499 }
500 }
501
502 /* Augmented data size is limited to sizeof(augmented_arg->unnamed union with value field) */
503 if (aug_size > value_size)
504 aug_size = value_size;
505
506 /* write data to payload */
507 if (augmented) {
508 int written = offsetof(struct augmented_arg, value) + aug_size;
509
510 if (written < 0 || written > sizeof(struct augmented_arg))
511 return 1;
512
513 ((struct augmented_arg *)payload_offset)->size = aug_size;
514 output += written;
515 payload_offset += written;
516 do_output = true;
517 }
518 }
519
520 if (!do_output || (sizeof(struct syscall_enter_args) + output) > sizeof(struct beauty_payload_enter))
521 return 1;
522
523 return augmented__beauty_output(ctx, payload, sizeof(struct syscall_enter_args) + output);
524 }
525
526 SEC("tp/raw_syscalls/sys_enter")
sys_enter(struct syscall_enter_args * args)527 int sys_enter(struct syscall_enter_args *args)
528 {
529 struct augmented_args_payload *augmented_args;
530 /*
531 * We start len, the amount of data that will be in the perf ring
532 * buffer, if this is not filtered out by one of pid_filter__has(),
533 * syscall->enabled, etc, with the non-augmented raw syscall payload,
534 * i.e. sizeof(augmented_args->args).
535 *
536 * We'll add to this as we add augmented syscalls right after that
537 * initial, non-augmented raw_syscalls:sys_enter payload.
538 */
539
540 if (pid_filter__has(&pids_filtered, getpid()))
541 return 0;
542
543 augmented_args = augmented_args_payload();
544 if (augmented_args == NULL)
545 return 1;
546
547 bpf_probe_read_kernel(&augmented_args->args, sizeof(augmented_args->args), args);
548
549 /*
550 * Jump to syscall specific augmenter, even if the default one,
551 * "!raw_syscalls:unaugmented" that will just return 1 to return the
552 * unaugmented tracepoint payload.
553 */
554 if (augment_sys_enter(args, &augmented_args->args))
555 bpf_tail_call(args, &syscalls_sys_enter, augmented_args->args.syscall_nr);
556
557 // If not found on the PROG_ARRAY syscalls map, then we're filtering it:
558 return 0;
559 }
560
561 SEC("tp/raw_syscalls/sys_exit")
sys_exit(struct syscall_exit_args * args)562 int sys_exit(struct syscall_exit_args *args)
563 {
564 struct syscall_exit_args exit_args;
565
566 if (pid_filter__has(&pids_filtered, getpid()))
567 return 0;
568
569 bpf_probe_read_kernel(&exit_args, sizeof(exit_args), args);
570 /*
571 * Jump to syscall specific return augmenter, even if the default one,
572 * "!raw_syscalls:unaugmented" that will just return 1 to return the
573 * unaugmented tracepoint payload.
574 */
575 bpf_tail_call(args, &syscalls_sys_exit, exit_args.syscall_nr);
576 /*
577 * If not found on the PROG_ARRAY syscalls map, then we're filtering it:
578 */
579 return 0;
580 }
581
582 char _license[] SEC("license") = "GPL";
583