1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2019 Facebook 3 #include <linux/sched.h> 4 #include <linux/ptrace.h> 5 #include <stdint.h> 6 #include <stddef.h> 7 #include <stdbool.h> 8 #include <linux/bpf.h> 9 #include "bpf_helpers.h" 10 11 #define FUNCTION_NAME_LEN 64 12 #define FILE_NAME_LEN 128 13 #define TASK_COMM_LEN 16 14 15 typedef struct { 16 int PyThreadState_frame; 17 int PyThreadState_thread; 18 int PyFrameObject_back; 19 int PyFrameObject_code; 20 int PyFrameObject_lineno; 21 int PyCodeObject_filename; 22 int PyCodeObject_name; 23 int String_data; 24 int String_size; 25 } OffsetConfig; 26 27 typedef struct { 28 uintptr_t current_state_addr; 29 uintptr_t tls_key_addr; 30 OffsetConfig offsets; 31 bool use_tls; 32 } PidData; 33 34 typedef struct { 35 uint32_t success; 36 } Stats; 37 38 typedef struct { 39 char name[FUNCTION_NAME_LEN]; 40 char file[FILE_NAME_LEN]; 41 } Symbol; 42 43 typedef struct { 44 uint32_t pid; 45 uint32_t tid; 46 char comm[TASK_COMM_LEN]; 47 int32_t kernel_stack_id; 48 int32_t user_stack_id; 49 bool thread_current; 50 bool pthread_match; 51 bool stack_complete; 52 int16_t stack_len; 53 int32_t stack[STACK_MAX_LEN]; 54 55 int has_meta; 56 int metadata; 57 char dummy_safeguard; 58 } Event; 59 60 61 struct bpf_elf_map { 62 __u32 type; 63 __u32 size_key; 64 __u32 size_value; 65 __u32 max_elem; 66 __u32 flags; 67 }; 68 69 typedef int pid_t; 70 71 typedef struct { 72 void* f_back; // PyFrameObject.f_back, previous frame 73 void* f_code; // PyFrameObject.f_code, pointer to PyCodeObject 74 void* co_filename; // PyCodeObject.co_filename 75 void* co_name; // PyCodeObject.co_name 76 } FrameData; 77 78 static inline __attribute__((__always_inline__)) void* 79 get_thread_state(void* tls_base, PidData* pidData) 80 { 81 void* thread_state; 82 int key; 83 84 bpf_probe_read(&key, sizeof(key), (void*)(long)pidData->tls_key_addr); 85 bpf_probe_read(&thread_state, sizeof(thread_state), 86 tls_base + 0x310 + key * 0x10 + 0x08); 87 return thread_state; 88 } 89 90 static inline __attribute__((__always_inline__)) bool 91 get_frame_data(void* frame_ptr, PidData* pidData, FrameData* frame, Symbol* symbol) 92 { 93 // read data from PyFrameObject 94 bpf_probe_read(&frame->f_back, 95 sizeof(frame->f_back), 96 frame_ptr + pidData->offsets.PyFrameObject_back); 97 bpf_probe_read(&frame->f_code, 98 sizeof(frame->f_code), 99 frame_ptr + pidData->offsets.PyFrameObject_code); 100 101 // read data from PyCodeObject 102 if (!frame->f_code) 103 return false; 104 bpf_probe_read(&frame->co_filename, 105 sizeof(frame->co_filename), 106 frame->f_code + pidData->offsets.PyCodeObject_filename); 107 bpf_probe_read(&frame->co_name, 108 sizeof(frame->co_name), 109 frame->f_code + pidData->offsets.PyCodeObject_name); 110 // read actual names into symbol 111 if (frame->co_filename) 112 bpf_probe_read_str(&symbol->file, 113 sizeof(symbol->file), 114 frame->co_filename + pidData->offsets.String_data); 115 if (frame->co_name) 116 bpf_probe_read_str(&symbol->name, 117 sizeof(symbol->name), 118 frame->co_name + pidData->offsets.String_data); 119 return true; 120 } 121 122 struct bpf_elf_map SEC("maps") pidmap = { 123 .type = BPF_MAP_TYPE_HASH, 124 .size_key = sizeof(int), 125 .size_value = sizeof(PidData), 126 .max_elem = 1, 127 }; 128 129 struct bpf_elf_map SEC("maps") eventmap = { 130 .type = BPF_MAP_TYPE_HASH, 131 .size_key = sizeof(int), 132 .size_value = sizeof(Event), 133 .max_elem = 1, 134 }; 135 136 struct bpf_elf_map SEC("maps") symbolmap = { 137 .type = BPF_MAP_TYPE_HASH, 138 .size_key = sizeof(Symbol), 139 .size_value = sizeof(int), 140 .max_elem = 1, 141 }; 142 143 struct bpf_elf_map SEC("maps") statsmap = { 144 .type = BPF_MAP_TYPE_ARRAY, 145 .size_key = sizeof(Stats), 146 .size_value = sizeof(int), 147 .max_elem = 1, 148 }; 149 150 struct bpf_elf_map SEC("maps") perfmap = { 151 .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY, 152 .size_key = sizeof(int), 153 .size_value = sizeof(int), 154 .max_elem = 32, 155 }; 156 157 struct bpf_elf_map SEC("maps") stackmap = { 158 .type = BPF_MAP_TYPE_STACK_TRACE, 159 .size_key = sizeof(int), 160 .size_value = sizeof(long long) * 127, 161 .max_elem = 1000, 162 }; 163 164 static inline __attribute__((__always_inline__)) int __on_event(struct pt_regs *ctx) 165 { 166 uint64_t pid_tgid = bpf_get_current_pid_tgid(); 167 pid_t pid = (pid_t)(pid_tgid >> 32); 168 PidData* pidData = bpf_map_lookup_elem(&pidmap, &pid); 169 if (!pidData) 170 return 0; 171 172 int zero = 0; 173 Event* event = bpf_map_lookup_elem(&eventmap, &zero); 174 if (!event) 175 return 0; 176 177 event->pid = pid; 178 179 event->tid = (pid_t)pid_tgid; 180 bpf_get_current_comm(&event->comm, sizeof(event->comm)); 181 182 event->user_stack_id = bpf_get_stackid(ctx, &stackmap, BPF_F_USER_STACK); 183 event->kernel_stack_id = bpf_get_stackid(ctx, &stackmap, 0); 184 185 void* thread_state_current = (void*)0; 186 bpf_probe_read(&thread_state_current, 187 sizeof(thread_state_current), 188 (void*)(long)pidData->current_state_addr); 189 190 struct task_struct* task = (struct task_struct*)bpf_get_current_task(); 191 void* tls_base = (void*)task; 192 193 void* thread_state = pidData->use_tls ? get_thread_state(tls_base, pidData) 194 : thread_state_current; 195 event->thread_current = thread_state == thread_state_current; 196 197 if (pidData->use_tls) { 198 uint64_t pthread_created; 199 uint64_t pthread_self; 200 bpf_probe_read(&pthread_self, sizeof(pthread_self), tls_base + 0x10); 201 202 bpf_probe_read(&pthread_created, 203 sizeof(pthread_created), 204 thread_state + pidData->offsets.PyThreadState_thread); 205 event->pthread_match = pthread_created == pthread_self; 206 } else { 207 event->pthread_match = 1; 208 } 209 210 if (event->pthread_match || !pidData->use_tls) { 211 void* frame_ptr; 212 FrameData frame; 213 Symbol sym = {}; 214 int cur_cpu = bpf_get_smp_processor_id(); 215 216 bpf_probe_read(&frame_ptr, 217 sizeof(frame_ptr), 218 thread_state + pidData->offsets.PyThreadState_frame); 219 220 int32_t* symbol_counter = bpf_map_lookup_elem(&symbolmap, &sym); 221 if (symbol_counter == NULL) 222 return 0; 223 #pragma unroll 224 /* Unwind python stack */ 225 for (int i = 0; i < STACK_MAX_LEN; ++i) { 226 if (frame_ptr && get_frame_data(frame_ptr, pidData, &frame, &sym)) { 227 int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu; 228 int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, &sym); 229 if (!symbol_id) { 230 bpf_map_update_elem(&symbolmap, &sym, &zero, 0); 231 symbol_id = bpf_map_lookup_elem(&symbolmap, &sym); 232 if (!symbol_id) 233 return 0; 234 } 235 if (*symbol_id == new_symbol_id) 236 (*symbol_counter)++; 237 event->stack[i] = *symbol_id; 238 event->stack_len = i + 1; 239 frame_ptr = frame.f_back; 240 } 241 } 242 event->stack_complete = frame_ptr == NULL; 243 } else { 244 event->stack_complete = 1; 245 } 246 247 Stats* stats = bpf_map_lookup_elem(&statsmap, &zero); 248 if (stats) 249 stats->success++; 250 251 event->has_meta = 0; 252 bpf_perf_event_output(ctx, &perfmap, 0, event, offsetof(Event, metadata)); 253 return 0; 254 } 255 256 SEC("raw_tracepoint/kfree_skb") 257 int on_event(struct pt_regs* ctx) 258 { 259 int i, ret = 0; 260 ret |= __on_event(ctx); 261 ret |= __on_event(ctx); 262 ret |= __on_event(ctx); 263 ret |= __on_event(ctx); 264 ret |= __on_event(ctx); 265 return ret; 266 } 267 268 char _license[] SEC("license") = "GPL"; 269