1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * bpf_kwork.c 4 * 5 * Copyright (c) 2022 Huawei Inc, Yang Jihong <yangjihong1@huawei.com> 6 */ 7 8 #include <time.h> 9 #include <fcntl.h> 10 #include <signal.h> 11 #include <stdio.h> 12 #include <unistd.h> 13 14 #include <linux/time64.h> 15 16 #include "util/debug.h" 17 #include "util/evsel.h" 18 #include "util/kwork.h" 19 20 #include <bpf/bpf.h> 21 #include <perf/cpumap.h> 22 23 #include "util/bpf_skel/kwork_trace.skel.h" 24 25 /* 26 * This should be in sync with "util/kwork_trace.bpf.c" 27 */ 28 #define MAX_KWORKNAME 128 29 30 struct work_key { 31 u32 type; 32 u32 cpu; 33 u64 id; 34 }; 35 36 struct report_data { 37 u64 nr; 38 u64 total_time; 39 u64 max_time; 40 u64 max_time_start; 41 u64 max_time_end; 42 }; 43 44 struct kwork_class_bpf { 45 struct kwork_class *class; 46 47 void (*load_prepare)(struct perf_kwork *kwork); 48 int (*get_work_name)(struct work_key *key, char **ret_name); 49 }; 50 51 static struct kwork_trace_bpf *skel; 52 53 static struct timespec ts_start; 54 static struct timespec ts_end; 55 56 void perf_kwork__trace_start(void) 57 { 58 clock_gettime(CLOCK_MONOTONIC, &ts_start); 59 skel->bss->enabled = 1; 60 } 61 62 void perf_kwork__trace_finish(void) 63 { 64 clock_gettime(CLOCK_MONOTONIC, &ts_end); 65 skel->bss->enabled = 0; 66 } 67 68 static int get_work_name_from_map(struct work_key *key, char **ret_name) 69 { 70 char name[MAX_KWORKNAME] = { 0 }; 71 int fd = bpf_map__fd(skel->maps.perf_kwork_names); 72 73 *ret_name = NULL; 74 75 if (fd < 0) { 76 pr_debug("Invalid names map fd\n"); 77 return 0; 78 } 79 80 if ((bpf_map_lookup_elem(fd, key, name) == 0) && (strlen(name) != 0)) { 81 *ret_name = strdup(name); 82 if (*ret_name == NULL) { 83 pr_err("Failed to copy work name\n"); 84 return -1; 85 } 86 } 87 88 return 0; 89 } 90 91 static void irq_load_prepare(struct perf_kwork *kwork) 92 { 93 if (kwork->report == KWORK_REPORT_RUNTIME) { 94 bpf_program__set_autoload(skel->progs.report_irq_handler_entry, true); 95 bpf_program__set_autoload(skel->progs.report_irq_handler_exit, true); 96 } 97 } 98 99 static struct kwork_class_bpf kwork_irq_bpf = { 100 .load_prepare = irq_load_prepare, 101 .get_work_name = get_work_name_from_map, 102 }; 103 104 static void softirq_load_prepare(struct perf_kwork *kwork) 105 { 106 if (kwork->report == KWORK_REPORT_RUNTIME) { 107 bpf_program__set_autoload(skel->progs.report_softirq_entry, true); 108 bpf_program__set_autoload(skel->progs.report_softirq_exit, true); 109 } else if (kwork->report == KWORK_REPORT_LATENCY) { 110 bpf_program__set_autoload(skel->progs.latency_softirq_raise, true); 111 bpf_program__set_autoload(skel->progs.latency_softirq_entry, true); 112 } 113 } 114 115 static struct kwork_class_bpf kwork_softirq_bpf = { 116 .load_prepare = softirq_load_prepare, 117 .get_work_name = get_work_name_from_map, 118 }; 119 120 static void workqueue_load_prepare(struct perf_kwork *kwork) 121 { 122 if (kwork->report == KWORK_REPORT_RUNTIME) { 123 bpf_program__set_autoload(skel->progs.report_workqueue_execute_start, true); 124 bpf_program__set_autoload(skel->progs.report_workqueue_execute_end, true); 125 } else if (kwork->report == KWORK_REPORT_LATENCY) { 126 bpf_program__set_autoload(skel->progs.latency_workqueue_activate_work, true); 127 bpf_program__set_autoload(skel->progs.latency_workqueue_execute_start, true); 128 } 129 } 130 131 static struct kwork_class_bpf kwork_workqueue_bpf = { 132 .load_prepare = workqueue_load_prepare, 133 .get_work_name = get_work_name_from_map, 134 }; 135 136 static struct kwork_class_bpf * 137 kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { 138 [KWORK_CLASS_IRQ] = &kwork_irq_bpf, 139 [KWORK_CLASS_SOFTIRQ] = &kwork_softirq_bpf, 140 [KWORK_CLASS_WORKQUEUE] = &kwork_workqueue_bpf, 141 }; 142 143 static bool valid_kwork_class_type(enum kwork_class_type type) 144 { 145 return type >= 0 && type < KWORK_CLASS_MAX ? true : false; 146 } 147 148 static int setup_filters(struct perf_kwork *kwork) 149 { 150 if (kwork->cpu_list != NULL) { 151 int idx, nr_cpus; 152 struct perf_cpu_map *map; 153 struct perf_cpu cpu; 154 int fd = bpf_map__fd(skel->maps.perf_kwork_cpu_filter); 155 156 if (fd < 0) { 157 pr_debug("Invalid cpu filter fd\n"); 158 return -1; 159 } 160 161 map = perf_cpu_map__new(kwork->cpu_list); 162 if (map == NULL) { 163 pr_debug("Invalid cpu_list\n"); 164 return -1; 165 } 166 167 nr_cpus = libbpf_num_possible_cpus(); 168 perf_cpu_map__for_each_cpu(cpu, idx, map) { 169 u8 val = 1; 170 171 if (cpu.cpu >= nr_cpus) { 172 perf_cpu_map__put(map); 173 pr_err("Requested cpu %d too large\n", cpu.cpu); 174 return -1; 175 } 176 bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY); 177 } 178 perf_cpu_map__put(map); 179 } 180 181 if (kwork->profile_name != NULL) { 182 int key, fd; 183 184 if (strlen(kwork->profile_name) >= MAX_KWORKNAME) { 185 pr_err("Requested name filter %s too large, limit to %d\n", 186 kwork->profile_name, MAX_KWORKNAME - 1); 187 return -1; 188 } 189 190 fd = bpf_map__fd(skel->maps.perf_kwork_name_filter); 191 if (fd < 0) { 192 pr_debug("Invalid name filter fd\n"); 193 return -1; 194 } 195 196 key = 0; 197 bpf_map_update_elem(fd, &key, kwork->profile_name, BPF_ANY); 198 } 199 200 return 0; 201 } 202 203 int perf_kwork__trace_prepare_bpf(struct perf_kwork *kwork) 204 { 205 struct bpf_program *prog; 206 struct kwork_class *class; 207 struct kwork_class_bpf *class_bpf; 208 enum kwork_class_type type; 209 210 skel = kwork_trace_bpf__open(); 211 if (!skel) { 212 pr_debug("Failed to open kwork trace skeleton\n"); 213 return -1; 214 } 215 216 /* 217 * set all progs to non-autoload, 218 * then set corresponding progs according to config 219 */ 220 bpf_object__for_each_program(prog, skel->obj) 221 bpf_program__set_autoload(prog, false); 222 223 list_for_each_entry(class, &kwork->class_list, list) { 224 type = class->type; 225 if (!valid_kwork_class_type(type) || 226 (kwork_class_bpf_supported_list[type] == NULL)) { 227 pr_err("Unsupported bpf trace class %s\n", class->name); 228 goto out; 229 } 230 231 class_bpf = kwork_class_bpf_supported_list[type]; 232 class_bpf->class = class; 233 234 if (class_bpf->load_prepare != NULL) 235 class_bpf->load_prepare(kwork); 236 } 237 238 if (kwork->cpu_list != NULL) 239 skel->rodata->has_cpu_filter = 1; 240 if (kwork->profile_name != NULL) 241 skel->rodata->has_name_filter = 1; 242 243 if (kwork_trace_bpf__load(skel)) { 244 pr_debug("Failed to load kwork trace skeleton\n"); 245 goto out; 246 } 247 248 if (setup_filters(kwork)) 249 goto out; 250 251 if (kwork_trace_bpf__attach(skel)) { 252 pr_debug("Failed to attach kwork trace skeleton\n"); 253 goto out; 254 } 255 256 return 0; 257 258 out: 259 kwork_trace_bpf__destroy(skel); 260 return -1; 261 } 262 263 static int add_work(struct perf_kwork *kwork, 264 struct work_key *key, 265 struct report_data *data) 266 { 267 struct kwork_work *work; 268 struct kwork_class_bpf *bpf_trace; 269 struct kwork_work tmp = { 270 .id = key->id, 271 .name = NULL, 272 .cpu = key->cpu, 273 }; 274 enum kwork_class_type type = key->type; 275 276 if (!valid_kwork_class_type(type)) { 277 pr_debug("Invalid class type %d to add work\n", type); 278 return -1; 279 } 280 281 bpf_trace = kwork_class_bpf_supported_list[type]; 282 tmp.class = bpf_trace->class; 283 284 if ((bpf_trace->get_work_name != NULL) && 285 (bpf_trace->get_work_name(key, &tmp.name))) 286 return -1; 287 288 work = perf_kwork_add_work(kwork, tmp.class, &tmp); 289 if (work == NULL) 290 return -1; 291 292 if (kwork->report == KWORK_REPORT_RUNTIME) { 293 work->nr_atoms = data->nr; 294 work->total_runtime = data->total_time; 295 work->max_runtime = data->max_time; 296 work->max_runtime_start = data->max_time_start; 297 work->max_runtime_end = data->max_time_end; 298 } else if (kwork->report == KWORK_REPORT_LATENCY) { 299 work->nr_atoms = data->nr; 300 work->total_latency = data->total_time; 301 work->max_latency = data->max_time; 302 work->max_latency_start = data->max_time_start; 303 work->max_latency_end = data->max_time_end; 304 } else { 305 pr_debug("Invalid bpf report type %d\n", kwork->report); 306 return -1; 307 } 308 309 kwork->timestart = (u64)ts_start.tv_sec * NSEC_PER_SEC + ts_start.tv_nsec; 310 kwork->timeend = (u64)ts_end.tv_sec * NSEC_PER_SEC + ts_end.tv_nsec; 311 312 return 0; 313 } 314 315 int perf_kwork__report_read_bpf(struct perf_kwork *kwork) 316 { 317 struct report_data data; 318 struct work_key key = { 319 .type = 0, 320 .cpu = 0, 321 .id = 0, 322 }; 323 struct work_key prev = { 324 .type = 0, 325 .cpu = 0, 326 .id = 0, 327 }; 328 int fd = bpf_map__fd(skel->maps.perf_kwork_report); 329 330 if (fd < 0) { 331 pr_debug("Invalid report fd\n"); 332 return -1; 333 } 334 335 while (!bpf_map_get_next_key(fd, &prev, &key)) { 336 if ((bpf_map_lookup_elem(fd, &key, &data)) != 0) { 337 pr_debug("Failed to lookup report elem\n"); 338 return -1; 339 } 340 341 if ((data.nr != 0) && (add_work(kwork, &key, &data) != 0)) 342 return -1; 343 344 prev = key; 345 } 346 return 0; 347 } 348 349 void perf_kwork__report_cleanup_bpf(void) 350 { 351 kwork_trace_bpf__destroy(skel); 352 } 353