1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (c) 2022, Huawei 3 4 #include "vmlinux.h" 5 #include <bpf/bpf_helpers.h> 6 #include <bpf/bpf_tracing.h> 7 8 #define KWORK_COUNT 100 9 #define MAX_KWORKNAME 128 10 11 /* 12 * This should be in sync with "util/kwork.h" 13 */ 14 enum kwork_class_type { 15 KWORK_CLASS_IRQ, 16 KWORK_CLASS_SOFTIRQ, 17 KWORK_CLASS_WORKQUEUE, 18 KWORK_CLASS_MAX, 19 }; 20 21 struct work_key { 22 __u32 type; 23 __u32 cpu; 24 __u64 id; 25 }; 26 27 struct report_data { 28 __u64 nr; 29 __u64 total_time; 30 __u64 max_time; 31 __u64 max_time_start; 32 __u64 max_time_end; 33 }; 34 35 struct { 36 __uint(type, BPF_MAP_TYPE_HASH); 37 __uint(key_size, sizeof(struct work_key)); 38 __uint(value_size, MAX_KWORKNAME); 39 __uint(max_entries, KWORK_COUNT); 40 } perf_kwork_names SEC(".maps"); 41 42 struct { 43 __uint(type, BPF_MAP_TYPE_HASH); 44 __uint(key_size, sizeof(struct work_key)); 45 __uint(value_size, sizeof(__u64)); 46 __uint(max_entries, KWORK_COUNT); 47 } perf_kwork_time SEC(".maps"); 48 49 struct { 50 __uint(type, BPF_MAP_TYPE_HASH); 51 __uint(key_size, sizeof(struct work_key)); 52 __uint(value_size, sizeof(struct report_data)); 53 __uint(max_entries, KWORK_COUNT); 54 } perf_kwork_report SEC(".maps"); 55 56 struct { 57 __uint(type, BPF_MAP_TYPE_HASH); 58 __uint(key_size, sizeof(__u32)); 59 __uint(value_size, sizeof(__u8)); 60 __uint(max_entries, 1); 61 } perf_kwork_cpu_filter SEC(".maps"); 62 63 struct { 64 __uint(type, BPF_MAP_TYPE_ARRAY); 65 __uint(key_size, sizeof(__u32)); 66 __uint(value_size, MAX_KWORKNAME); 67 __uint(max_entries, 1); 68 } perf_kwork_name_filter SEC(".maps"); 69 70 int enabled = 0; 71 int has_cpu_filter = 0; 72 int has_name_filter = 0; 73 74 static __always_inline int local_strncmp(const char *s1, 75 unsigned int sz, const char *s2) 76 { 77 int ret = 0; 78 unsigned int i; 79 80 for (i = 0; i < sz; i++) { 81 ret = (unsigned char)s1[i] - (unsigned char)s2[i]; 82 if (ret || !s1[i] || !s2[i]) 83 break; 84 } 85 86 return ret; 87 } 88 89 static __always_inline int trace_event_match(struct work_key *key, char *name) 90 { 91 __u8 *cpu_val; 92 char *name_val; 93 __u32 zero = 0; 94 __u32 cpu = bpf_get_smp_processor_id(); 95 96 if (!enabled) 97 return 0; 98 99 if (has_cpu_filter) { 100 cpu_val = bpf_map_lookup_elem(&perf_kwork_cpu_filter, &cpu); 101 if (!cpu_val) 102 return 0; 103 } 104 105 if (has_name_filter && (name != NULL)) { 106 name_val = bpf_map_lookup_elem(&perf_kwork_name_filter, &zero); 107 if (name_val && 108 (local_strncmp(name_val, MAX_KWORKNAME, name) != 0)) { 109 return 0; 110 } 111 } 112 113 return 1; 114 } 115 116 static __always_inline void do_update_time(void *map, struct work_key *key, 117 __u64 time_start, __u64 time_end) 118 { 119 struct report_data zero, *data; 120 __s64 delta = time_end - time_start; 121 122 if (delta < 0) 123 return; 124 125 data = bpf_map_lookup_elem(map, key); 126 if (!data) { 127 __builtin_memset(&zero, 0, sizeof(zero)); 128 bpf_map_update_elem(map, key, &zero, BPF_NOEXIST); 129 data = bpf_map_lookup_elem(map, key); 130 if (!data) 131 return; 132 } 133 134 if ((delta > data->max_time) || 135 (data->max_time == 0)) { 136 data->max_time = delta; 137 data->max_time_start = time_start; 138 data->max_time_end = time_end; 139 } 140 141 data->total_time += delta; 142 data->nr++; 143 } 144 145 static __always_inline void do_update_timestart(void *map, struct work_key *key) 146 { 147 __u64 ts = bpf_ktime_get_ns(); 148 149 bpf_map_update_elem(map, key, &ts, BPF_ANY); 150 } 151 152 static __always_inline void do_update_timeend(void *report_map, void *time_map, 153 struct work_key *key) 154 { 155 __u64 *time = bpf_map_lookup_elem(time_map, key); 156 157 if (time) { 158 bpf_map_delete_elem(time_map, key); 159 do_update_time(report_map, key, *time, bpf_ktime_get_ns()); 160 } 161 } 162 163 static __always_inline void do_update_name(void *map, 164 struct work_key *key, char *name) 165 { 166 if (!bpf_map_lookup_elem(map, key)) 167 bpf_map_update_elem(map, key, name, BPF_ANY); 168 } 169 170 static __always_inline int update_timestart(void *map, struct work_key *key) 171 { 172 if (!trace_event_match(key, NULL)) 173 return 0; 174 175 do_update_timestart(map, key); 176 return 0; 177 } 178 179 static __always_inline int update_timestart_and_name(void *time_map, 180 void *names_map, 181 struct work_key *key, 182 char *name) 183 { 184 if (!trace_event_match(key, name)) 185 return 0; 186 187 do_update_timestart(time_map, key); 188 do_update_name(names_map, key, name); 189 190 return 0; 191 } 192 193 static __always_inline int update_timeend(void *report_map, 194 void *time_map, struct work_key *key) 195 { 196 if (!trace_event_match(key, NULL)) 197 return 0; 198 199 do_update_timeend(report_map, time_map, key); 200 201 return 0; 202 } 203 204 static __always_inline int update_timeend_and_name(void *report_map, 205 void *time_map, 206 void *names_map, 207 struct work_key *key, 208 char *name) 209 { 210 if (!trace_event_match(key, name)) 211 return 0; 212 213 do_update_timeend(report_map, time_map, key); 214 do_update_name(names_map, key, name); 215 216 return 0; 217 } 218 219 SEC("tracepoint/irq/irq_handler_entry") 220 int report_irq_handler_entry(struct trace_event_raw_irq_handler_entry *ctx) 221 { 222 char name[MAX_KWORKNAME]; 223 struct work_key key = { 224 .type = KWORK_CLASS_IRQ, 225 .cpu = bpf_get_smp_processor_id(), 226 .id = (__u64)ctx->irq, 227 }; 228 void *name_addr = (void *)ctx + (ctx->__data_loc_name & 0xffff); 229 230 bpf_probe_read_kernel_str(name, sizeof(name), name_addr); 231 232 return update_timestart_and_name(&perf_kwork_time, 233 &perf_kwork_names, &key, name); 234 } 235 236 SEC("tracepoint/irq/irq_handler_exit") 237 int report_irq_handler_exit(struct trace_event_raw_irq_handler_exit *ctx) 238 { 239 struct work_key key = { 240 .type = KWORK_CLASS_IRQ, 241 .cpu = bpf_get_smp_processor_id(), 242 .id = (__u64)ctx->irq, 243 }; 244 245 return update_timeend(&perf_kwork_report, &perf_kwork_time, &key); 246 } 247 248 static char softirq_name_list[NR_SOFTIRQS][MAX_KWORKNAME] = { 249 { "HI" }, 250 { "TIMER" }, 251 { "NET_TX" }, 252 { "NET_RX" }, 253 { "BLOCK" }, 254 { "IRQ_POLL" }, 255 { "TASKLET" }, 256 { "SCHED" }, 257 { "HRTIMER" }, 258 { "RCU" }, 259 }; 260 261 SEC("tracepoint/irq/softirq_entry") 262 int report_softirq_entry(struct trace_event_raw_softirq *ctx) 263 { 264 unsigned int vec = ctx->vec; 265 struct work_key key = { 266 .type = KWORK_CLASS_SOFTIRQ, 267 .cpu = bpf_get_smp_processor_id(), 268 .id = (__u64)vec, 269 }; 270 271 if (vec < NR_SOFTIRQS) { 272 return update_timestart_and_name(&perf_kwork_time, 273 &perf_kwork_names, &key, 274 softirq_name_list[vec]); 275 } 276 277 return 0; 278 } 279 280 SEC("tracepoint/irq/softirq_exit") 281 int report_softirq_exit(struct trace_event_raw_softirq *ctx) 282 { 283 struct work_key key = { 284 .type = KWORK_CLASS_SOFTIRQ, 285 .cpu = bpf_get_smp_processor_id(), 286 .id = (__u64)ctx->vec, 287 }; 288 289 return update_timeend(&perf_kwork_report, &perf_kwork_time, &key); 290 } 291 292 SEC("tracepoint/irq/softirq_raise") 293 int latency_softirq_raise(struct trace_event_raw_softirq *ctx) 294 { 295 unsigned int vec = ctx->vec; 296 struct work_key key = { 297 .type = KWORK_CLASS_SOFTIRQ, 298 .cpu = bpf_get_smp_processor_id(), 299 .id = (__u64)vec, 300 }; 301 302 if (vec < NR_SOFTIRQS) { 303 return update_timestart_and_name(&perf_kwork_time, 304 &perf_kwork_names, &key, 305 softirq_name_list[vec]); 306 } 307 308 return 0; 309 } 310 311 SEC("tracepoint/irq/softirq_entry") 312 int latency_softirq_entry(struct trace_event_raw_softirq *ctx) 313 { 314 struct work_key key = { 315 .type = KWORK_CLASS_SOFTIRQ, 316 .cpu = bpf_get_smp_processor_id(), 317 .id = (__u64)ctx->vec, 318 }; 319 320 return update_timeend(&perf_kwork_report, &perf_kwork_time, &key); 321 } 322 323 SEC("tracepoint/workqueue/workqueue_execute_start") 324 int report_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx) 325 { 326 struct work_key key = { 327 .type = KWORK_CLASS_WORKQUEUE, 328 .cpu = bpf_get_smp_processor_id(), 329 .id = (__u64)ctx->work, 330 }; 331 332 return update_timestart(&perf_kwork_time, &key); 333 } 334 335 SEC("tracepoint/workqueue/workqueue_execute_end") 336 int report_workqueue_execute_end(struct trace_event_raw_workqueue_execute_end *ctx) 337 { 338 char name[MAX_KWORKNAME]; 339 struct work_key key = { 340 .type = KWORK_CLASS_WORKQUEUE, 341 .cpu = bpf_get_smp_processor_id(), 342 .id = (__u64)ctx->work, 343 }; 344 unsigned long long func_addr = (unsigned long long)ctx->function; 345 346 __builtin_memset(name, 0, sizeof(name)); 347 bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr)); 348 349 return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time, 350 &perf_kwork_names, &key, name); 351 } 352 353 SEC("tracepoint/workqueue/workqueue_activate_work") 354 int latency_workqueue_activate_work(struct trace_event_raw_workqueue_activate_work *ctx) 355 { 356 struct work_key key = { 357 .type = KWORK_CLASS_WORKQUEUE, 358 .cpu = bpf_get_smp_processor_id(), 359 .id = (__u64)ctx->work, 360 }; 361 362 return update_timestart(&perf_kwork_time, &key); 363 } 364 365 SEC("tracepoint/workqueue/workqueue_execute_start") 366 int latency_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx) 367 { 368 char name[MAX_KWORKNAME]; 369 struct work_key key = { 370 .type = KWORK_CLASS_WORKQUEUE, 371 .cpu = bpf_get_smp_processor_id(), 372 .id = (__u64)ctx->work, 373 }; 374 unsigned long long func_addr = (unsigned long long)ctx->function; 375 376 __builtin_memset(name, 0, sizeof(name)); 377 bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr)); 378 379 return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time, 380 &perf_kwork_names, &key, name); 381 } 382 383 char LICENSE[] SEC("license") = "Dual BSD/GPL"; 384