1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * bpf_kwork.c 4 * 5 * Copyright (c) 2022 Huawei Inc, Yang Jihong <yangjihong1@huawei.com> 6 */ 7 8 #include <time.h> 9 #include <fcntl.h> 10 #include <signal.h> 11 #include <stdio.h> 12 #include <unistd.h> 13 14 #include <linux/time64.h> 15 16 #include "util/debug.h" 17 #include "util/evsel.h" 18 #include "util/kwork.h" 19 20 #include <bpf/bpf.h> 21 #include <perf/cpumap.h> 22 23 #include "util/bpf_skel/kwork_trace.skel.h" 24 25 /* 26 * This should be in sync with "util/kwork_trace.bpf.c" 27 */ 28 #define MAX_KWORKNAME 128 29 30 struct work_key { 31 u32 type; 32 u32 cpu; 33 u64 id; 34 }; 35 36 struct report_data { 37 u64 nr; 38 u64 total_time; 39 u64 max_time; 40 u64 max_time_start; 41 u64 max_time_end; 42 }; 43 44 struct kwork_class_bpf { 45 struct kwork_class *class; 46 47 void (*load_prepare)(struct perf_kwork *kwork); 48 int (*get_work_name)(struct work_key *key, char **ret_name); 49 }; 50 51 static struct kwork_trace_bpf *skel; 52 53 static struct timespec ts_start; 54 static struct timespec ts_end; 55 56 void perf_kwork__trace_start(void) 57 { 58 clock_gettime(CLOCK_MONOTONIC, &ts_start); 59 skel->bss->enabled = 1; 60 } 61 62 void perf_kwork__trace_finish(void) 63 { 64 clock_gettime(CLOCK_MONOTONIC, &ts_end); 65 skel->bss->enabled = 0; 66 } 67 68 static int get_work_name_from_map(struct work_key *key, char **ret_name) 69 { 70 char name[MAX_KWORKNAME] = { 0 }; 71 int fd = bpf_map__fd(skel->maps.perf_kwork_names); 72 73 *ret_name = NULL; 74 75 if (fd < 0) { 76 pr_debug("Invalid names map fd\n"); 77 return 0; 78 } 79 80 if ((bpf_map_lookup_elem(fd, key, name) == 0) && (strlen(name) != 0)) { 81 *ret_name = strdup(name); 82 if (*ret_name == NULL) { 83 pr_err("Failed to copy work name\n"); 84 return -1; 85 } 86 } 87 88 return 0; 89 } 90 91 static void irq_load_prepare(struct perf_kwork *kwork) 92 { 93 if (kwork->report == KWORK_REPORT_RUNTIME) { 94 bpf_program__set_autoload(skel->progs.report_irq_handler_entry, true); 95 bpf_program__set_autoload(skel->progs.report_irq_handler_exit, true); 96 } 97 } 98 99 static struct kwork_class_bpf kwork_irq_bpf = { 100 .load_prepare = irq_load_prepare, 101 .get_work_name = get_work_name_from_map, 102 }; 103 104 static void softirq_load_prepare(struct perf_kwork *kwork) 105 { 106 if (kwork->report == KWORK_REPORT_RUNTIME) { 107 bpf_program__set_autoload(skel->progs.report_softirq_entry, true); 108 bpf_program__set_autoload(skel->progs.report_softirq_exit, true); 109 } else if (kwork->report == KWORK_REPORT_LATENCY) { 110 bpf_program__set_autoload(skel->progs.latency_softirq_raise, true); 111 bpf_program__set_autoload(skel->progs.latency_softirq_entry, true); 112 } 113 } 114 115 static struct kwork_class_bpf kwork_softirq_bpf = { 116 .load_prepare = softirq_load_prepare, 117 .get_work_name = get_work_name_from_map, 118 }; 119 120 static void workqueue_load_prepare(struct perf_kwork *kwork) 121 { 122 if (kwork->report == KWORK_REPORT_RUNTIME) { 123 bpf_program__set_autoload(skel->progs.report_workqueue_execute_start, true); 124 bpf_program__set_autoload(skel->progs.report_workqueue_execute_end, true); 125 } else if (kwork->report == KWORK_REPORT_LATENCY) { 126 bpf_program__set_autoload(skel->progs.latency_workqueue_activate_work, true); 127 bpf_program__set_autoload(skel->progs.latency_workqueue_execute_start, true); 128 } 129 } 130 131 static struct kwork_class_bpf kwork_workqueue_bpf = { 132 .load_prepare = workqueue_load_prepare, 133 .get_work_name = get_work_name_from_map, 134 }; 135 136 static struct kwork_class_bpf * 137 kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { 138 [KWORK_CLASS_IRQ] = &kwork_irq_bpf, 139 [KWORK_CLASS_SOFTIRQ] = &kwork_softirq_bpf, 140 [KWORK_CLASS_WORKQUEUE] = &kwork_workqueue_bpf, 141 }; 142 143 static bool valid_kwork_class_type(enum kwork_class_type type) 144 { 145 return type >= 0 && type < KWORK_CLASS_MAX ? true : false; 146 } 147 148 static int setup_filters(struct perf_kwork *kwork) 149 { 150 if (kwork->cpu_list != NULL) { 151 int idx, nr_cpus; 152 struct perf_cpu_map *map; 153 struct perf_cpu cpu; 154 int fd = bpf_map__fd(skel->maps.perf_kwork_cpu_filter); 155 156 if (fd < 0) { 157 pr_debug("Invalid cpu filter fd\n"); 158 return -1; 159 } 160 161 map = perf_cpu_map__new(kwork->cpu_list); 162 if (map == NULL) { 163 pr_debug("Invalid cpu_list\n"); 164 return -1; 165 } 166 167 nr_cpus = libbpf_num_possible_cpus(); 168 perf_cpu_map__for_each_cpu(cpu, idx, map) { 169 u8 val = 1; 170 171 if (cpu.cpu >= nr_cpus) { 172 perf_cpu_map__put(map); 173 pr_err("Requested cpu %d too large\n", cpu.cpu); 174 return -1; 175 } 176 bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY); 177 } 178 perf_cpu_map__put(map); 179 180 skel->bss->has_cpu_filter = 1; 181 } 182 183 if (kwork->profile_name != NULL) { 184 int key, fd; 185 186 if (strlen(kwork->profile_name) >= MAX_KWORKNAME) { 187 pr_err("Requested name filter %s too large, limit to %d\n", 188 kwork->profile_name, MAX_KWORKNAME - 1); 189 return -1; 190 } 191 192 fd = bpf_map__fd(skel->maps.perf_kwork_name_filter); 193 if (fd < 0) { 194 pr_debug("Invalid name filter fd\n"); 195 return -1; 196 } 197 198 key = 0; 199 bpf_map_update_elem(fd, &key, kwork->profile_name, BPF_ANY); 200 201 skel->bss->has_name_filter = 1; 202 } 203 204 return 0; 205 } 206 207 int perf_kwork__trace_prepare_bpf(struct perf_kwork *kwork) 208 { 209 struct bpf_program *prog; 210 struct kwork_class *class; 211 struct kwork_class_bpf *class_bpf; 212 enum kwork_class_type type; 213 214 skel = kwork_trace_bpf__open(); 215 if (!skel) { 216 pr_debug("Failed to open kwork trace skeleton\n"); 217 return -1; 218 } 219 220 /* 221 * set all progs to non-autoload, 222 * then set corresponding progs according to config 223 */ 224 bpf_object__for_each_program(prog, skel->obj) 225 bpf_program__set_autoload(prog, false); 226 227 list_for_each_entry(class, &kwork->class_list, list) { 228 type = class->type; 229 if (!valid_kwork_class_type(type) || 230 (kwork_class_bpf_supported_list[type] == NULL)) { 231 pr_err("Unsupported bpf trace class %s\n", class->name); 232 goto out; 233 } 234 235 class_bpf = kwork_class_bpf_supported_list[type]; 236 class_bpf->class = class; 237 238 if (class_bpf->load_prepare != NULL) 239 class_bpf->load_prepare(kwork); 240 } 241 242 if (kwork_trace_bpf__load(skel)) { 243 pr_debug("Failed to load kwork trace skeleton\n"); 244 goto out; 245 } 246 247 if (setup_filters(kwork)) 248 goto out; 249 250 if (kwork_trace_bpf__attach(skel)) { 251 pr_debug("Failed to attach kwork trace skeleton\n"); 252 goto out; 253 } 254 255 return 0; 256 257 out: 258 kwork_trace_bpf__destroy(skel); 259 return -1; 260 } 261 262 static int add_work(struct perf_kwork *kwork, 263 struct work_key *key, 264 struct report_data *data) 265 { 266 struct kwork_work *work; 267 struct kwork_class_bpf *bpf_trace; 268 struct kwork_work tmp = { 269 .id = key->id, 270 .name = NULL, 271 .cpu = key->cpu, 272 }; 273 enum kwork_class_type type = key->type; 274 275 if (!valid_kwork_class_type(type)) { 276 pr_debug("Invalid class type %d to add work\n", type); 277 return -1; 278 } 279 280 bpf_trace = kwork_class_bpf_supported_list[type]; 281 tmp.class = bpf_trace->class; 282 283 if ((bpf_trace->get_work_name != NULL) && 284 (bpf_trace->get_work_name(key, &tmp.name))) 285 return -1; 286 287 work = perf_kwork_add_work(kwork, tmp.class, &tmp); 288 if (work == NULL) 289 return -1; 290 291 if (kwork->report == KWORK_REPORT_RUNTIME) { 292 work->nr_atoms = data->nr; 293 work->total_runtime = data->total_time; 294 work->max_runtime = data->max_time; 295 work->max_runtime_start = data->max_time_start; 296 work->max_runtime_end = data->max_time_end; 297 } else if (kwork->report == KWORK_REPORT_LATENCY) { 298 work->nr_atoms = data->nr; 299 work->total_latency = data->total_time; 300 work->max_latency = data->max_time; 301 work->max_latency_start = data->max_time_start; 302 work->max_latency_end = data->max_time_end; 303 } else { 304 pr_debug("Invalid bpf report type %d\n", kwork->report); 305 return -1; 306 } 307 308 kwork->timestart = (u64)ts_start.tv_sec * NSEC_PER_SEC + ts_start.tv_nsec; 309 kwork->timeend = (u64)ts_end.tv_sec * NSEC_PER_SEC + ts_end.tv_nsec; 310 311 return 0; 312 } 313 314 int perf_kwork__report_read_bpf(struct perf_kwork *kwork) 315 { 316 struct report_data data; 317 struct work_key key = { 318 .type = 0, 319 .cpu = 0, 320 .id = 0, 321 }; 322 struct work_key prev = { 323 .type = 0, 324 .cpu = 0, 325 .id = 0, 326 }; 327 int fd = bpf_map__fd(skel->maps.perf_kwork_report); 328 329 if (fd < 0) { 330 pr_debug("Invalid report fd\n"); 331 return -1; 332 } 333 334 while (!bpf_map_get_next_key(fd, &prev, &key)) { 335 if ((bpf_map_lookup_elem(fd, &key, &data)) != 0) { 336 pr_debug("Failed to lookup report elem\n"); 337 return -1; 338 } 339 340 if ((data.nr != 0) && (add_work(kwork, &key, &data) != 0)) 341 return -1; 342 343 prev = key; 344 } 345 return 0; 346 } 347 348 void perf_kwork__report_cleanup_bpf(void) 349 { 350 kwork_trace_bpf__destroy(skel); 351 } 352