1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2019 Facebook 3 #include <linux/sched.h> 4 #include <linux/ptrace.h> 5 #include <stdint.h> 6 #include <stddef.h> 7 #include <stdbool.h> 8 #include <linux/bpf.h> 9 #include "bpf_helpers.h" 10 11 #define FUNCTION_NAME_LEN 64 12 #define FILE_NAME_LEN 128 13 #define TASK_COMM_LEN 16 14 15 typedef struct { 16 int PyThreadState_frame; 17 int PyThreadState_thread; 18 int PyFrameObject_back; 19 int PyFrameObject_code; 20 int PyFrameObject_lineno; 21 int PyCodeObject_filename; 22 int PyCodeObject_name; 23 int String_data; 24 int String_size; 25 } OffsetConfig; 26 27 typedef struct { 28 uintptr_t current_state_addr; 29 uintptr_t tls_key_addr; 30 OffsetConfig offsets; 31 bool use_tls; 32 } PidData; 33 34 typedef struct { 35 uint32_t success; 36 } Stats; 37 38 typedef struct { 39 char name[FUNCTION_NAME_LEN]; 40 char file[FILE_NAME_LEN]; 41 } Symbol; 42 43 typedef struct { 44 uint32_t pid; 45 uint32_t tid; 46 char comm[TASK_COMM_LEN]; 47 int32_t kernel_stack_id; 48 int32_t user_stack_id; 49 bool thread_current; 50 bool pthread_match; 51 bool stack_complete; 52 int16_t stack_len; 53 int32_t stack[STACK_MAX_LEN]; 54 55 int has_meta; 56 int metadata; 57 char dummy_safeguard; 58 } Event; 59 60 61 struct bpf_elf_map { 62 __u32 type; 63 __u32 size_key; 64 __u32 size_value; 65 __u32 max_elem; 66 __u32 flags; 67 }; 68 69 typedef int pid_t; 70 71 typedef struct { 72 void* f_back; // PyFrameObject.f_back, previous frame 73 void* f_code; // PyFrameObject.f_code, pointer to PyCodeObject 74 void* co_filename; // PyCodeObject.co_filename 75 void* co_name; // PyCodeObject.co_name 76 } FrameData; 77 78 static __always_inline void *get_thread_state(void *tls_base, PidData *pidData) 79 { 80 void* thread_state; 81 int key; 82 83 bpf_probe_read(&key, sizeof(key), (void*)(long)pidData->tls_key_addr); 84 bpf_probe_read(&thread_state, sizeof(thread_state), 85 tls_base + 0x310 + key * 0x10 + 0x08); 86 return thread_state; 87 } 88 89 static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData, 90 FrameData *frame, Symbol *symbol) 91 { 92 // read data from PyFrameObject 93 bpf_probe_read(&frame->f_back, 94 sizeof(frame->f_back), 95 frame_ptr + pidData->offsets.PyFrameObject_back); 96 bpf_probe_read(&frame->f_code, 97 sizeof(frame->f_code), 98 frame_ptr + pidData->offsets.PyFrameObject_code); 99 100 // read data from PyCodeObject 101 if (!frame->f_code) 102 return false; 103 bpf_probe_read(&frame->co_filename, 104 sizeof(frame->co_filename), 105 frame->f_code + pidData->offsets.PyCodeObject_filename); 106 bpf_probe_read(&frame->co_name, 107 sizeof(frame->co_name), 108 frame->f_code + pidData->offsets.PyCodeObject_name); 109 // read actual names into symbol 110 if (frame->co_filename) 111 bpf_probe_read_str(&symbol->file, 112 sizeof(symbol->file), 113 frame->co_filename + pidData->offsets.String_data); 114 if (frame->co_name) 115 bpf_probe_read_str(&symbol->name, 116 sizeof(symbol->name), 117 frame->co_name + pidData->offsets.String_data); 118 return true; 119 } 120 121 struct bpf_elf_map SEC("maps") pidmap = { 122 .type = BPF_MAP_TYPE_HASH, 123 .size_key = sizeof(int), 124 .size_value = sizeof(PidData), 125 .max_elem = 1, 126 }; 127 128 struct bpf_elf_map SEC("maps") eventmap = { 129 .type = BPF_MAP_TYPE_HASH, 130 .size_key = sizeof(int), 131 .size_value = sizeof(Event), 132 .max_elem = 1, 133 }; 134 135 struct bpf_elf_map SEC("maps") symbolmap = { 136 .type = BPF_MAP_TYPE_HASH, 137 .size_key = sizeof(Symbol), 138 .size_value = sizeof(int), 139 .max_elem = 1, 140 }; 141 142 struct bpf_elf_map SEC("maps") statsmap = { 143 .type = BPF_MAP_TYPE_ARRAY, 144 .size_key = sizeof(Stats), 145 .size_value = sizeof(int), 146 .max_elem = 1, 147 }; 148 149 struct bpf_elf_map SEC("maps") perfmap = { 150 .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY, 151 .size_key = sizeof(int), 152 .size_value = sizeof(int), 153 .max_elem = 32, 154 }; 155 156 struct bpf_elf_map SEC("maps") stackmap = { 157 .type = BPF_MAP_TYPE_STACK_TRACE, 158 .size_key = sizeof(int), 159 .size_value = sizeof(long long) * 127, 160 .max_elem = 1000, 161 }; 162 163 static __always_inline int __on_event(struct pt_regs *ctx) 164 { 165 uint64_t pid_tgid = bpf_get_current_pid_tgid(); 166 pid_t pid = (pid_t)(pid_tgid >> 32); 167 PidData* pidData = bpf_map_lookup_elem(&pidmap, &pid); 168 if (!pidData) 169 return 0; 170 171 int zero = 0; 172 Event* event = bpf_map_lookup_elem(&eventmap, &zero); 173 if (!event) 174 return 0; 175 176 event->pid = pid; 177 178 event->tid = (pid_t)pid_tgid; 179 bpf_get_current_comm(&event->comm, sizeof(event->comm)); 180 181 event->user_stack_id = bpf_get_stackid(ctx, &stackmap, BPF_F_USER_STACK); 182 event->kernel_stack_id = bpf_get_stackid(ctx, &stackmap, 0); 183 184 void* thread_state_current = (void*)0; 185 bpf_probe_read(&thread_state_current, 186 sizeof(thread_state_current), 187 (void*)(long)pidData->current_state_addr); 188 189 struct task_struct* task = (struct task_struct*)bpf_get_current_task(); 190 void* tls_base = (void*)task; 191 192 void* thread_state = pidData->use_tls ? get_thread_state(tls_base, pidData) 193 : thread_state_current; 194 event->thread_current = thread_state == thread_state_current; 195 196 if (pidData->use_tls) { 197 uint64_t pthread_created; 198 uint64_t pthread_self; 199 bpf_probe_read(&pthread_self, sizeof(pthread_self), tls_base + 0x10); 200 201 bpf_probe_read(&pthread_created, 202 sizeof(pthread_created), 203 thread_state + pidData->offsets.PyThreadState_thread); 204 event->pthread_match = pthread_created == pthread_self; 205 } else { 206 event->pthread_match = 1; 207 } 208 209 if (event->pthread_match || !pidData->use_tls) { 210 void* frame_ptr; 211 FrameData frame; 212 Symbol sym = {}; 213 int cur_cpu = bpf_get_smp_processor_id(); 214 215 bpf_probe_read(&frame_ptr, 216 sizeof(frame_ptr), 217 thread_state + pidData->offsets.PyThreadState_frame); 218 219 int32_t* symbol_counter = bpf_map_lookup_elem(&symbolmap, &sym); 220 if (symbol_counter == NULL) 221 return 0; 222 #ifdef NO_UNROLL 223 #pragma clang loop unroll(disable) 224 #else 225 #pragma clang loop unroll(full) 226 #endif 227 /* Unwind python stack */ 228 for (int i = 0; i < STACK_MAX_LEN; ++i) { 229 if (frame_ptr && get_frame_data(frame_ptr, pidData, &frame, &sym)) { 230 int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu; 231 int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, &sym); 232 if (!symbol_id) { 233 bpf_map_update_elem(&symbolmap, &sym, &zero, 0); 234 symbol_id = bpf_map_lookup_elem(&symbolmap, &sym); 235 if (!symbol_id) 236 return 0; 237 } 238 if (*symbol_id == new_symbol_id) 239 (*symbol_counter)++; 240 event->stack[i] = *symbol_id; 241 event->stack_len = i + 1; 242 frame_ptr = frame.f_back; 243 } 244 } 245 event->stack_complete = frame_ptr == NULL; 246 } else { 247 event->stack_complete = 1; 248 } 249 250 Stats* stats = bpf_map_lookup_elem(&statsmap, &zero); 251 if (stats) 252 stats->success++; 253 254 event->has_meta = 0; 255 bpf_perf_event_output(ctx, &perfmap, 0, event, offsetof(Event, metadata)); 256 return 0; 257 } 258 259 SEC("raw_tracepoint/kfree_skb") 260 int on_event(struct pt_regs* ctx) 261 { 262 int i, ret = 0; 263 ret |= __on_event(ctx); 264 ret |= __on_event(ctx); 265 ret |= __on_event(ctx); 266 ret |= __on_event(ctx); 267 ret |= __on_event(ctx); 268 return ret; 269 } 270 271 char _license[] SEC("license") = "GPL"; 272