1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2019 Facebook
3 #include <linux/sched.h>
4 #include <linux/ptrace.h>
5 #include <stdint.h>
6 #include <stddef.h>
7 #include <stdbool.h>
8 #include <linux/bpf.h>
9 #include <bpf/bpf_helpers.h>
10 #include "bpf_misc.h"
11 #include "bpf_compiler.h"
12
13 #define FUNCTION_NAME_LEN 64
14 #define FILE_NAME_LEN 128
15 #define TASK_COMM_LEN 16
16
17 typedef struct {
18 int PyThreadState_frame;
19 int PyThreadState_thread;
20 int PyFrameObject_back;
21 int PyFrameObject_code;
22 int PyFrameObject_lineno;
23 int PyCodeObject_filename;
24 int PyCodeObject_name;
25 int String_data;
26 int String_size;
27 } OffsetConfig;
28
29 typedef struct {
30 uintptr_t current_state_addr;
31 uintptr_t tls_key_addr;
32 OffsetConfig offsets;
33 bool use_tls;
34 } PidData;
35
36 typedef struct {
37 uint32_t success;
38 } Stats;
39
40 typedef struct {
41 char name[FUNCTION_NAME_LEN];
42 char file[FILE_NAME_LEN];
43 } Symbol;
44
45 typedef struct {
46 uint32_t pid;
47 uint32_t tid;
48 char comm[TASK_COMM_LEN];
49 int32_t kernel_stack_id;
50 int32_t user_stack_id;
51 bool thread_current;
52 bool pthread_match;
53 bool stack_complete;
54 int16_t stack_len;
55 int32_t stack[STACK_MAX_LEN];
56
57 int has_meta;
58 int metadata;
59 char dummy_safeguard;
60 } Event;
61
62
63 typedef int pid_t;
64
65 typedef struct {
66 void* f_back; // PyFrameObject.f_back, previous frame
67 void* f_code; // PyFrameObject.f_code, pointer to PyCodeObject
68 void* co_filename; // PyCodeObject.co_filename
69 void* co_name; // PyCodeObject.co_name
70 } FrameData;
71
72 #ifdef SUBPROGS
73 __noinline
74 #else
75 __always_inline
76 #endif
get_thread_state(void * tls_base,PidData * pidData)77 static void *get_thread_state(void *tls_base, PidData *pidData)
78 {
79 void* thread_state;
80 int key;
81
82 bpf_probe_read_user(&key, sizeof(key), (void*)(long)pidData->tls_key_addr);
83 bpf_probe_read_user(&thread_state, sizeof(thread_state),
84 tls_base + 0x310 + key * 0x10 + 0x08);
85 return thread_state;
86 }
87
__get_frame_data(long frame_ptr_,PidData * pidData __arg_nonnull,FrameData * frame __arg_nonnull,Symbol * symbol __arg_nonnull)88 __weak bool __get_frame_data(long frame_ptr_, PidData *pidData __arg_nonnull,
89 FrameData *frame __arg_nonnull, Symbol *symbol __arg_nonnull)
90 {
91 void *frame_ptr = (void *)frame_ptr_;
92
93 // read data from PyFrameObject
94 bpf_probe_read_user(&frame->f_back,
95 sizeof(frame->f_back),
96 frame_ptr + pidData->offsets.PyFrameObject_back);
97 bpf_probe_read_user(&frame->f_code,
98 sizeof(frame->f_code),
99 frame_ptr + pidData->offsets.PyFrameObject_code);
100
101 // read data from PyCodeObject
102 if (!frame->f_code)
103 return false;
104 bpf_probe_read_user(&frame->co_filename,
105 sizeof(frame->co_filename),
106 frame->f_code + pidData->offsets.PyCodeObject_filename);
107 bpf_probe_read_user(&frame->co_name,
108 sizeof(frame->co_name),
109 frame->f_code + pidData->offsets.PyCodeObject_name);
110 // read actual names into symbol
111 if (frame->co_filename)
112 bpf_probe_read_user_str(&symbol->file,
113 sizeof(symbol->file),
114 frame->co_filename +
115 pidData->offsets.String_data);
116 if (frame->co_name)
117 bpf_probe_read_user_str(&symbol->name,
118 sizeof(symbol->name),
119 frame->co_name +
120 pidData->offsets.String_data);
121 return true;
122 }
123
get_frame_data(void * frame_ptr,PidData * pidData,FrameData * frame,Symbol * symbol)124 static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData,
125 FrameData *frame, Symbol *symbol)
126 {
127 return __get_frame_data((long)frame_ptr, pidData, frame, symbol);
128 }
129
130 struct {
131 __uint(type, BPF_MAP_TYPE_HASH);
132 __uint(max_entries, 1);
133 __type(key, int);
134 __type(value, PidData);
135 } pidmap SEC(".maps");
136
137 struct {
138 __uint(type, BPF_MAP_TYPE_HASH);
139 __uint(max_entries, 1);
140 __type(key, int);
141 __type(value, Event);
142 } eventmap SEC(".maps");
143
144 struct {
145 __uint(type, BPF_MAP_TYPE_HASH);
146 __uint(max_entries, 1);
147 __type(key, Symbol);
148 __type(value, int);
149 } symbolmap SEC(".maps");
150
151 struct {
152 __uint(type, BPF_MAP_TYPE_ARRAY);
153 __uint(max_entries, 1);
154 __type(key, int);
155 __type(value, Stats);
156 } statsmap SEC(".maps");
157
158 struct {
159 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
160 __uint(max_entries, 32);
161 __uint(key_size, sizeof(int));
162 __uint(value_size, sizeof(int));
163 } perfmap SEC(".maps");
164
165 struct {
166 __uint(type, BPF_MAP_TYPE_STACK_TRACE);
167 __uint(max_entries, 1000);
168 __uint(key_size, sizeof(int));
169 __uint(value_size, sizeof(long long) * 127);
170 } stackmap SEC(".maps");
171
172 #ifdef USE_BPF_LOOP
173 struct process_frame_ctx {
174 int cur_cpu;
175 int32_t *symbol_counter;
176 void *frame_ptr;
177 FrameData *frame;
178 PidData *pidData;
179 Symbol *sym;
180 Event *event;
181 bool done;
182 };
183
process_frame_callback(__u32 i,struct process_frame_ctx * ctx)184 static int process_frame_callback(__u32 i, struct process_frame_ctx *ctx)
185 {
186 int zero = 0;
187 void *frame_ptr = ctx->frame_ptr;
188 PidData *pidData = ctx->pidData;
189 FrameData *frame = ctx->frame;
190 int32_t *symbol_counter = ctx->symbol_counter;
191 int cur_cpu = ctx->cur_cpu;
192 Event *event = ctx->event;
193 Symbol *sym = ctx->sym;
194
195 if (frame_ptr && get_frame_data(frame_ptr, pidData, frame, sym)) {
196 int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu;
197 int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, sym);
198
199 if (!symbol_id) {
200 bpf_map_update_elem(&symbolmap, sym, &zero, 0);
201 symbol_id = bpf_map_lookup_elem(&symbolmap, sym);
202 if (!symbol_id) {
203 ctx->done = true;
204 return 1;
205 }
206 }
207 if (*symbol_id == new_symbol_id)
208 (*symbol_counter)++;
209
210 barrier_var(i);
211 if (i >= STACK_MAX_LEN)
212 return 1;
213
214 event->stack[i] = *symbol_id;
215
216 event->stack_len = i + 1;
217 frame_ptr = frame->f_back;
218 }
219 return 0;
220 }
221 #endif /* USE_BPF_LOOP */
222
223 #ifdef GLOBAL_FUNC
224 __noinline
225 #elif defined(SUBPROGS)
226 static __noinline
227 #else
228 static __always_inline
229 #endif
__on_event(struct bpf_raw_tracepoint_args * ctx)230 int __on_event(struct bpf_raw_tracepoint_args *ctx)
231 {
232 uint64_t pid_tgid = bpf_get_current_pid_tgid();
233 pid_t pid = (pid_t)(pid_tgid >> 32);
234 PidData* pidData = bpf_map_lookup_elem(&pidmap, &pid);
235 if (!pidData)
236 return 0;
237
238 int zero = 0;
239 Event* event = bpf_map_lookup_elem(&eventmap, &zero);
240 if (!event)
241 return 0;
242
243 event->pid = pid;
244
245 event->tid = (pid_t)pid_tgid;
246 bpf_get_current_comm(&event->comm, sizeof(event->comm));
247
248 event->user_stack_id = bpf_get_stackid(ctx, &stackmap, BPF_F_USER_STACK);
249 event->kernel_stack_id = bpf_get_stackid(ctx, &stackmap, 0);
250
251 void* thread_state_current = (void*)0;
252 bpf_probe_read_user(&thread_state_current,
253 sizeof(thread_state_current),
254 (void*)(long)pidData->current_state_addr);
255
256 struct task_struct* task = (struct task_struct*)bpf_get_current_task();
257 void* tls_base = (void*)task;
258
259 void* thread_state = pidData->use_tls ? get_thread_state(tls_base, pidData)
260 : thread_state_current;
261 event->thread_current = thread_state == thread_state_current;
262
263 if (pidData->use_tls) {
264 uint64_t pthread_created;
265 uint64_t pthread_self;
266 bpf_probe_read_user(&pthread_self, sizeof(pthread_self),
267 tls_base + 0x10);
268
269 bpf_probe_read_user(&pthread_created,
270 sizeof(pthread_created),
271 thread_state +
272 pidData->offsets.PyThreadState_thread);
273 event->pthread_match = pthread_created == pthread_self;
274 } else {
275 event->pthread_match = 1;
276 }
277
278 if (event->pthread_match || !pidData->use_tls) {
279 void* frame_ptr;
280 FrameData frame;
281 Symbol sym = {};
282 int cur_cpu = bpf_get_smp_processor_id();
283
284 bpf_probe_read_user(&frame_ptr,
285 sizeof(frame_ptr),
286 thread_state +
287 pidData->offsets.PyThreadState_frame);
288
289 int32_t* symbol_counter = bpf_map_lookup_elem(&symbolmap, &sym);
290 if (symbol_counter == NULL)
291 return 0;
292 #ifdef USE_BPF_LOOP
293 struct process_frame_ctx ctx = {
294 .cur_cpu = cur_cpu,
295 .symbol_counter = symbol_counter,
296 .frame_ptr = frame_ptr,
297 .frame = &frame,
298 .pidData = pidData,
299 .sym = &sym,
300 .event = event,
301 };
302
303 bpf_loop(STACK_MAX_LEN, process_frame_callback, &ctx, 0);
304 if (ctx.done)
305 return 0;
306 #else
307 #if defined(USE_ITER)
308 /* no for loop, no unrolling */
309 #elif defined(NO_UNROLL)
310 __pragma_loop_no_unroll
311 #elif defined(UNROLL_COUNT)
312 __pragma_loop_unroll_count(UNROLL_COUNT)
313 #else
314 __pragma_loop_unroll_full
315 #endif /* NO_UNROLL */
316 /* Unwind python stack */
317 #ifdef USE_ITER
318 int i;
319 bpf_for(i, 0, STACK_MAX_LEN) {
320 #else /* !USE_ITER */
321 for (int i = 0; i < STACK_MAX_LEN; ++i) {
322 #endif
323 if (frame_ptr && get_frame_data(frame_ptr, pidData, &frame, &sym)) {
324 int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu;
325 int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
326 if (!symbol_id) {
327 bpf_map_update_elem(&symbolmap, &sym, &zero, 0);
328 symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
329 if (!symbol_id)
330 return 0;
331 }
332 if (*symbol_id == new_symbol_id)
333 (*symbol_counter)++;
334 event->stack[i] = *symbol_id;
335 event->stack_len = i + 1;
336 frame_ptr = frame.f_back;
337 }
338 }
339 #endif /* USE_BPF_LOOP */
340 event->stack_complete = frame_ptr == NULL;
341 } else {
342 event->stack_complete = 1;
343 }
344
345 Stats* stats = bpf_map_lookup_elem(&statsmap, &zero);
346 if (stats)
347 stats->success++;
348
349 event->has_meta = 0;
350 bpf_perf_event_output(ctx, &perfmap, 0, event, offsetof(Event, metadata));
351 return 0;
352 }
353
354 SEC("raw_tracepoint/kfree_skb")
355 int on_event(struct bpf_raw_tracepoint_args* ctx)
356 {
357 int ret = 0;
358 ret |= __on_event(ctx);
359 ret |= __on_event(ctx);
360 ret |= __on_event(ctx);
361 ret |= __on_event(ctx);
362 ret |= __on_event(ctx);
363 return ret;
364 }
365
366 char _license[] SEC("license") = "GPL";
367