1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2019 Facebook 3 #include <linux/sched.h> 4 #include <linux/ptrace.h> 5 #include <stdint.h> 6 #include <stddef.h> 7 #include <stdbool.h> 8 #include <linux/bpf.h> 9 #include <bpf/bpf_helpers.h> 10 #include "bpf_misc.h" 11 #include "bpf_compiler.h" 12 13 #define FUNCTION_NAME_LEN 64 14 #define FILE_NAME_LEN 128 15 #define TASK_COMM_LEN 16 16 17 typedef struct { 18 int PyThreadState_frame; 19 int PyThreadState_thread; 20 int PyFrameObject_back; 21 int PyFrameObject_code; 22 int PyFrameObject_lineno; 23 int PyCodeObject_filename; 24 int PyCodeObject_name; 25 int String_data; 26 int String_size; 27 } OffsetConfig; 28 29 typedef struct { 30 uintptr_t current_state_addr; 31 uintptr_t tls_key_addr; 32 OffsetConfig offsets; 33 bool use_tls; 34 } PidData; 35 36 typedef struct { 37 uint32_t success; 38 } Stats; 39 40 typedef struct { 41 char name[FUNCTION_NAME_LEN]; 42 char file[FILE_NAME_LEN]; 43 } Symbol; 44 45 typedef struct { 46 uint32_t pid; 47 uint32_t tid; 48 char comm[TASK_COMM_LEN]; 49 int32_t kernel_stack_id; 50 int32_t user_stack_id; 51 bool thread_current; 52 bool pthread_match; 53 bool stack_complete; 54 int16_t stack_len; 55 int32_t stack[STACK_MAX_LEN]; 56 57 int has_meta; 58 int metadata; 59 char dummy_safeguard; 60 } Event; 61 62 63 typedef int pid_t; 64 65 typedef struct { 66 void* f_back; // PyFrameObject.f_back, previous frame 67 void* f_code; // PyFrameObject.f_code, pointer to PyCodeObject 68 void* co_filename; // PyCodeObject.co_filename 69 void* co_name; // PyCodeObject.co_name 70 } FrameData; 71 72 #ifdef SUBPROGS 73 __noinline 74 #else 75 __always_inline 76 #endif 77 static void *get_thread_state(void *tls_base, PidData *pidData) 78 { 79 void* thread_state; 80 int key; 81 82 bpf_probe_read_user(&key, sizeof(key), (void*)(long)pidData->tls_key_addr); 83 bpf_probe_read_user(&thread_state, sizeof(thread_state), 84 tls_base + 0x310 + key * 0x10 + 0x08); 85 return thread_state; 86 } 87 88 __weak bool __get_frame_data(long frame_ptr_, PidData *pidData __arg_nonnull, 89 FrameData *frame __arg_nonnull, Symbol *symbol __arg_nonnull) 90 { 91 void *frame_ptr = (void *)frame_ptr_; 92 93 // read data from PyFrameObject 94 bpf_probe_read_user(&frame->f_back, 95 sizeof(frame->f_back), 96 frame_ptr + pidData->offsets.PyFrameObject_back); 97 bpf_probe_read_user(&frame->f_code, 98 sizeof(frame->f_code), 99 frame_ptr + pidData->offsets.PyFrameObject_code); 100 101 // read data from PyCodeObject 102 if (!frame->f_code) 103 return false; 104 bpf_probe_read_user(&frame->co_filename, 105 sizeof(frame->co_filename), 106 frame->f_code + pidData->offsets.PyCodeObject_filename); 107 bpf_probe_read_user(&frame->co_name, 108 sizeof(frame->co_name), 109 frame->f_code + pidData->offsets.PyCodeObject_name); 110 // read actual names into symbol 111 if (frame->co_filename) 112 bpf_probe_read_user_str(&symbol->file, 113 sizeof(symbol->file), 114 frame->co_filename + 115 pidData->offsets.String_data); 116 if (frame->co_name) 117 bpf_probe_read_user_str(&symbol->name, 118 sizeof(symbol->name), 119 frame->co_name + 120 pidData->offsets.String_data); 121 return true; 122 } 123 124 static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData, 125 FrameData *frame, Symbol *symbol) 126 { 127 return __get_frame_data((long)frame_ptr, pidData, frame, symbol); 128 } 129 130 struct { 131 __uint(type, BPF_MAP_TYPE_HASH); 132 __uint(max_entries, 1); 133 __type(key, int); 134 __type(value, PidData); 135 } pidmap SEC(".maps"); 136 137 struct { 138 __uint(type, BPF_MAP_TYPE_HASH); 139 __uint(max_entries, 1); 140 __type(key, int); 141 __type(value, Event); 142 } eventmap SEC(".maps"); 143 144 struct { 145 __uint(type, BPF_MAP_TYPE_HASH); 146 __uint(max_entries, 1); 147 __type(key, Symbol); 148 __type(value, int); 149 } symbolmap SEC(".maps"); 150 151 struct { 152 __uint(type, BPF_MAP_TYPE_ARRAY); 153 __uint(max_entries, 1); 154 __type(key, int); 155 __type(value, Stats); 156 } statsmap SEC(".maps"); 157 158 struct { 159 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); 160 __uint(max_entries, 32); 161 __uint(key_size, sizeof(int)); 162 __uint(value_size, sizeof(int)); 163 } perfmap SEC(".maps"); 164 165 struct { 166 __uint(type, BPF_MAP_TYPE_STACK_TRACE); 167 __uint(max_entries, 1000); 168 __uint(key_size, sizeof(int)); 169 __uint(value_size, sizeof(long long) * 127); 170 } stackmap SEC(".maps"); 171 172 #ifdef USE_BPF_LOOP 173 struct process_frame_ctx { 174 int cur_cpu; 175 int32_t *symbol_counter; 176 void *frame_ptr; 177 FrameData *frame; 178 PidData *pidData; 179 Symbol *sym; 180 Event *event; 181 bool done; 182 }; 183 184 static int process_frame_callback(__u32 i, struct process_frame_ctx *ctx) 185 { 186 int zero = 0; 187 void *frame_ptr = ctx->frame_ptr; 188 PidData *pidData = ctx->pidData; 189 FrameData *frame = ctx->frame; 190 int32_t *symbol_counter = ctx->symbol_counter; 191 int cur_cpu = ctx->cur_cpu; 192 Event *event = ctx->event; 193 Symbol *sym = ctx->sym; 194 195 if (frame_ptr && get_frame_data(frame_ptr, pidData, frame, sym)) { 196 int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu; 197 int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, sym); 198 199 if (!symbol_id) { 200 bpf_map_update_elem(&symbolmap, sym, &zero, 0); 201 symbol_id = bpf_map_lookup_elem(&symbolmap, sym); 202 if (!symbol_id) { 203 ctx->done = true; 204 return 1; 205 } 206 } 207 if (*symbol_id == new_symbol_id) 208 (*symbol_counter)++; 209 210 barrier_var(i); 211 if (i >= STACK_MAX_LEN) 212 return 1; 213 214 event->stack[i] = *symbol_id; 215 216 event->stack_len = i + 1; 217 frame_ptr = frame->f_back; 218 } 219 return 0; 220 } 221 #endif /* USE_BPF_LOOP */ 222 223 #ifdef GLOBAL_FUNC 224 __noinline 225 #elif defined(SUBPROGS) 226 static __noinline 227 #else 228 static __always_inline 229 #endif 230 int __on_event(struct bpf_raw_tracepoint_args *ctx) 231 { 232 uint64_t pid_tgid = bpf_get_current_pid_tgid(); 233 pid_t pid = (pid_t)(pid_tgid >> 32); 234 PidData* pidData = bpf_map_lookup_elem(&pidmap, &pid); 235 if (!pidData) 236 return 0; 237 238 int zero = 0; 239 Event* event = bpf_map_lookup_elem(&eventmap, &zero); 240 if (!event) 241 return 0; 242 243 event->pid = pid; 244 245 event->tid = (pid_t)pid_tgid; 246 bpf_get_current_comm(&event->comm, sizeof(event->comm)); 247 248 event->user_stack_id = bpf_get_stackid(ctx, &stackmap, BPF_F_USER_STACK); 249 event->kernel_stack_id = bpf_get_stackid(ctx, &stackmap, 0); 250 251 void* thread_state_current = (void*)0; 252 bpf_probe_read_user(&thread_state_current, 253 sizeof(thread_state_current), 254 (void*)(long)pidData->current_state_addr); 255 256 struct task_struct* task = (struct task_struct*)bpf_get_current_task(); 257 void* tls_base = (void*)task; 258 259 void* thread_state = pidData->use_tls ? get_thread_state(tls_base, pidData) 260 : thread_state_current; 261 event->thread_current = thread_state == thread_state_current; 262 263 if (pidData->use_tls) { 264 uint64_t pthread_created; 265 uint64_t pthread_self; 266 bpf_probe_read_user(&pthread_self, sizeof(pthread_self), 267 tls_base + 0x10); 268 269 bpf_probe_read_user(&pthread_created, 270 sizeof(pthread_created), 271 thread_state + 272 pidData->offsets.PyThreadState_thread); 273 event->pthread_match = pthread_created == pthread_self; 274 } else { 275 event->pthread_match = 1; 276 } 277 278 if (event->pthread_match || !pidData->use_tls) { 279 void* frame_ptr; 280 FrameData frame; 281 Symbol sym = {}; 282 int cur_cpu = bpf_get_smp_processor_id(); 283 284 bpf_probe_read_user(&frame_ptr, 285 sizeof(frame_ptr), 286 thread_state + 287 pidData->offsets.PyThreadState_frame); 288 289 int32_t* symbol_counter = bpf_map_lookup_elem(&symbolmap, &sym); 290 if (symbol_counter == NULL) 291 return 0; 292 #ifdef USE_BPF_LOOP 293 struct process_frame_ctx ctx = { 294 .cur_cpu = cur_cpu, 295 .symbol_counter = symbol_counter, 296 .frame_ptr = frame_ptr, 297 .frame = &frame, 298 .pidData = pidData, 299 .sym = &sym, 300 .event = event, 301 }; 302 303 bpf_loop(STACK_MAX_LEN, process_frame_callback, &ctx, 0); 304 if (ctx.done) 305 return 0; 306 #else 307 #if defined(USE_ITER) 308 /* no for loop, no unrolling */ 309 #elif defined(NO_UNROLL) 310 __pragma_loop_no_unroll 311 #elif defined(UNROLL_COUNT) 312 __pragma_loop_unroll_count(UNROLL_COUNT) 313 #else 314 __pragma_loop_unroll_full 315 #endif /* NO_UNROLL */ 316 /* Unwind python stack */ 317 #ifdef USE_ITER 318 int i; 319 bpf_for(i, 0, STACK_MAX_LEN) { 320 #else /* !USE_ITER */ 321 for (int i = 0; i < STACK_MAX_LEN; ++i) { 322 #endif 323 if (frame_ptr && get_frame_data(frame_ptr, pidData, &frame, &sym)) { 324 int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu; 325 int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, &sym); 326 if (!symbol_id) { 327 bpf_map_update_elem(&symbolmap, &sym, &zero, 0); 328 symbol_id = bpf_map_lookup_elem(&symbolmap, &sym); 329 if (!symbol_id) 330 return 0; 331 } 332 if (*symbol_id == new_symbol_id) 333 (*symbol_counter)++; 334 event->stack[i] = *symbol_id; 335 event->stack_len = i + 1; 336 frame_ptr = frame.f_back; 337 } 338 } 339 #endif /* USE_BPF_LOOP */ 340 event->stack_complete = frame_ptr == NULL; 341 } else { 342 event->stack_complete = 1; 343 } 344 345 Stats* stats = bpf_map_lookup_elem(&statsmap, &zero); 346 if (stats) 347 stats->success++; 348 349 event->has_meta = 0; 350 bpf_perf_event_output(ctx, &perfmap, 0, event, offsetof(Event, metadata)); 351 return 0; 352 } 353 354 SEC("raw_tracepoint/kfree_skb") 355 int on_event(struct bpf_raw_tracepoint_args* ctx) 356 { 357 int ret = 0; 358 ret |= __on_event(ctx); 359 ret |= __on_event(ctx); 360 ret |= __on_event(ctx); 361 ret |= __on_event(ctx); 362 ret |= __on_event(ctx); 363 return ret; 364 } 365 366 char _license[] SEC("license") = "GPL"; 367