1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * bpf_kwork.c 4 * 5 * Copyright (c) 2022 Huawei Inc, Yang Jihong <yangjihong1@huawei.com> 6 */ 7 8 #include <time.h> 9 #include <fcntl.h> 10 #include <signal.h> 11 #include <stdio.h> 12 #include <unistd.h> 13 14 #include <linux/time64.h> 15 16 #include "util/debug.h" 17 #include "util/evsel.h" 18 #include "util/kwork.h" 19 20 #include <bpf/bpf.h> 21 #include <perf/cpumap.h> 22 23 #include "util/bpf_skel/kwork_trace.skel.h" 24 25 /* 26 * This should be in sync with "util/kwork_trace.bpf.c" 27 */ 28 #define MAX_KWORKNAME 128 29 30 struct work_key { 31 u32 type; 32 u32 cpu; 33 u64 id; 34 }; 35 36 struct report_data { 37 u64 nr; 38 u64 total_time; 39 u64 max_time; 40 u64 max_time_start; 41 u64 max_time_end; 42 }; 43 44 struct kwork_class_bpf { 45 struct kwork_class *class; 46 47 void (*load_prepare)(struct perf_kwork *kwork); 48 int (*get_work_name)(struct work_key *key, char **ret_name); 49 }; 50 51 static struct kwork_trace_bpf *skel; 52 53 static struct timespec ts_start; 54 static struct timespec ts_end; 55 56 void perf_kwork__trace_start(void) 57 { 58 clock_gettime(CLOCK_MONOTONIC, &ts_start); 59 skel->bss->enabled = 1; 60 } 61 62 void perf_kwork__trace_finish(void) 63 { 64 clock_gettime(CLOCK_MONOTONIC, &ts_end); 65 skel->bss->enabled = 0; 66 } 67 68 static int get_work_name_from_map(struct work_key *key, char **ret_name) 69 { 70 char name[MAX_KWORKNAME] = { 0 }; 71 int fd = bpf_map__fd(skel->maps.perf_kwork_names); 72 73 *ret_name = NULL; 74 75 if (fd < 0) { 76 pr_debug("Invalid names map fd\n"); 77 return 0; 78 } 79 80 if ((bpf_map_lookup_elem(fd, key, name) == 0) && (strlen(name) != 0)) { 81 *ret_name = strdup(name); 82 if (*ret_name == NULL) { 83 pr_err("Failed to copy work name\n"); 84 return -1; 85 } 86 } 87 88 return 0; 89 } 90 91 static void irq_load_prepare(struct perf_kwork *kwork) 92 { 93 if (kwork->report == KWORK_REPORT_RUNTIME) { 94 bpf_program__set_autoload(skel->progs.report_irq_handler_entry, true); 95 bpf_program__set_autoload(skel->progs.report_irq_handler_exit, true); 96 } 97 } 98 99 static struct kwork_class_bpf kwork_irq_bpf = { 100 .load_prepare = irq_load_prepare, 101 .get_work_name = get_work_name_from_map, 102 }; 103 104 static void softirq_load_prepare(struct perf_kwork *kwork) 105 { 106 if (kwork->report == KWORK_REPORT_RUNTIME) { 107 bpf_program__set_autoload(skel->progs.report_softirq_entry, true); 108 bpf_program__set_autoload(skel->progs.report_softirq_exit, true); 109 } else if (kwork->report == KWORK_REPORT_LATENCY) { 110 bpf_program__set_autoload(skel->progs.latency_softirq_raise, true); 111 bpf_program__set_autoload(skel->progs.latency_softirq_entry, true); 112 } 113 } 114 115 static struct kwork_class_bpf kwork_softirq_bpf = { 116 .load_prepare = softirq_load_prepare, 117 .get_work_name = get_work_name_from_map, 118 }; 119 120 static void workqueue_load_prepare(struct perf_kwork *kwork) 121 { 122 if (kwork->report == KWORK_REPORT_RUNTIME) { 123 bpf_program__set_autoload(skel->progs.report_workqueue_execute_start, true); 124 bpf_program__set_autoload(skel->progs.report_workqueue_execute_end, true); 125 } else if (kwork->report == KWORK_REPORT_LATENCY) { 126 bpf_program__set_autoload(skel->progs.latency_workqueue_activate_work, true); 127 bpf_program__set_autoload(skel->progs.latency_workqueue_execute_start, true); 128 } 129 } 130 131 static struct kwork_class_bpf kwork_workqueue_bpf = { 132 .load_prepare = workqueue_load_prepare, 133 .get_work_name = get_work_name_from_map, 134 }; 135 136 static struct kwork_class_bpf * 137 kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { 138 [KWORK_CLASS_IRQ] = &kwork_irq_bpf, 139 [KWORK_CLASS_SOFTIRQ] = &kwork_softirq_bpf, 140 [KWORK_CLASS_WORKQUEUE] = &kwork_workqueue_bpf, 141 }; 142 143 static bool valid_kwork_class_type(enum kwork_class_type type) 144 { 145 return type >= 0 && type < KWORK_CLASS_MAX ? true : false; 146 } 147 148 static int setup_filters(struct perf_kwork *kwork) 149 { 150 if (kwork->cpu_list != NULL) { 151 unsigned int idx; 152 int nr_cpus; 153 struct perf_cpu_map *map; 154 struct perf_cpu cpu; 155 int fd = bpf_map__fd(skel->maps.perf_kwork_cpu_filter); 156 157 if (fd < 0) { 158 pr_debug("Invalid cpu filter fd\n"); 159 return -1; 160 } 161 162 map = perf_cpu_map__new(kwork->cpu_list); 163 if (map == NULL) { 164 pr_debug("Invalid cpu_list\n"); 165 return -1; 166 } 167 168 nr_cpus = libbpf_num_possible_cpus(); 169 perf_cpu_map__for_each_cpu(cpu, idx, map) { 170 u8 val = 1; 171 172 if (cpu.cpu >= nr_cpus) { 173 perf_cpu_map__put(map); 174 pr_err("Requested cpu %d too large\n", cpu.cpu); 175 return -1; 176 } 177 bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY); 178 } 179 perf_cpu_map__put(map); 180 } 181 182 if (kwork->profile_name != NULL) { 183 int key, fd; 184 185 if (strlen(kwork->profile_name) >= MAX_KWORKNAME) { 186 pr_err("Requested name filter %s too large, limit to %d\n", 187 kwork->profile_name, MAX_KWORKNAME - 1); 188 return -1; 189 } 190 191 fd = bpf_map__fd(skel->maps.perf_kwork_name_filter); 192 if (fd < 0) { 193 pr_debug("Invalid name filter fd\n"); 194 return -1; 195 } 196 197 key = 0; 198 bpf_map_update_elem(fd, &key, kwork->profile_name, BPF_ANY); 199 } 200 201 return 0; 202 } 203 204 int perf_kwork__trace_prepare_bpf(struct perf_kwork *kwork) 205 { 206 struct bpf_program *prog; 207 struct kwork_class *class; 208 struct kwork_class_bpf *class_bpf; 209 enum kwork_class_type type; 210 211 skel = kwork_trace_bpf__open(); 212 if (!skel) { 213 pr_debug("Failed to open kwork trace skeleton\n"); 214 return -1; 215 } 216 217 /* 218 * set all progs to non-autoload, 219 * then set corresponding progs according to config 220 */ 221 bpf_object__for_each_program(prog, skel->obj) 222 bpf_program__set_autoload(prog, false); 223 224 list_for_each_entry(class, &kwork->class_list, list) { 225 type = class->type; 226 if (!valid_kwork_class_type(type) || 227 (kwork_class_bpf_supported_list[type] == NULL)) { 228 pr_err("Unsupported bpf trace class %s\n", class->name); 229 goto out; 230 } 231 232 class_bpf = kwork_class_bpf_supported_list[type]; 233 class_bpf->class = class; 234 235 if (class_bpf->load_prepare != NULL) 236 class_bpf->load_prepare(kwork); 237 } 238 239 if (kwork->cpu_list != NULL) 240 skel->rodata->has_cpu_filter = 1; 241 if (kwork->profile_name != NULL) 242 skel->rodata->has_name_filter = 1; 243 244 if (kwork_trace_bpf__load(skel)) { 245 pr_debug("Failed to load kwork trace skeleton\n"); 246 goto out; 247 } 248 249 if (setup_filters(kwork)) 250 goto out; 251 252 if (kwork_trace_bpf__attach(skel)) { 253 pr_debug("Failed to attach kwork trace skeleton\n"); 254 goto out; 255 } 256 257 return 0; 258 259 out: 260 kwork_trace_bpf__destroy(skel); 261 return -1; 262 } 263 264 static int add_work(struct perf_kwork *kwork, 265 struct work_key *key, 266 struct report_data *data) 267 { 268 struct kwork_work *work; 269 struct kwork_class_bpf *bpf_trace; 270 struct kwork_work tmp = { 271 .id = key->id, 272 .name = NULL, 273 .cpu = key->cpu, 274 }; 275 enum kwork_class_type type = key->type; 276 277 if (!valid_kwork_class_type(type)) { 278 pr_debug("Invalid class type %d to add work\n", type); 279 return -1; 280 } 281 282 bpf_trace = kwork_class_bpf_supported_list[type]; 283 tmp.class = bpf_trace->class; 284 285 if ((bpf_trace->get_work_name != NULL) && 286 (bpf_trace->get_work_name(key, &tmp.name))) 287 return -1; 288 289 work = kwork->add_work(kwork, tmp.class, &tmp); 290 if (work == NULL) 291 return -1; 292 293 if (kwork->report == KWORK_REPORT_RUNTIME) { 294 work->nr_atoms = data->nr; 295 work->total_runtime = data->total_time; 296 work->max_runtime = data->max_time; 297 work->max_runtime_start = data->max_time_start; 298 work->max_runtime_end = data->max_time_end; 299 } else if (kwork->report == KWORK_REPORT_LATENCY) { 300 work->nr_atoms = data->nr; 301 work->total_latency = data->total_time; 302 work->max_latency = data->max_time; 303 work->max_latency_start = data->max_time_start; 304 work->max_latency_end = data->max_time_end; 305 } else { 306 pr_debug("Invalid bpf report type %d\n", kwork->report); 307 return -1; 308 } 309 310 kwork->timestart = (u64)ts_start.tv_sec * NSEC_PER_SEC + ts_start.tv_nsec; 311 kwork->timeend = (u64)ts_end.tv_sec * NSEC_PER_SEC + ts_end.tv_nsec; 312 313 return 0; 314 } 315 316 int perf_kwork__report_read_bpf(struct perf_kwork *kwork) 317 { 318 struct report_data data; 319 struct work_key key = { 320 .type = 0, 321 .cpu = 0, 322 .id = 0, 323 }; 324 struct work_key prev = { 325 .type = 0, 326 .cpu = 0, 327 .id = 0, 328 }; 329 int fd = bpf_map__fd(skel->maps.perf_kwork_report); 330 331 if (fd < 0) { 332 pr_debug("Invalid report fd\n"); 333 return -1; 334 } 335 336 while (!bpf_map_get_next_key(fd, &prev, &key)) { 337 if ((bpf_map_lookup_elem(fd, &key, &data)) != 0) { 338 pr_debug("Failed to lookup report elem\n"); 339 return -1; 340 } 341 342 if ((data.nr != 0) && (add_work(kwork, &key, &data) != 0)) 343 return -1; 344 345 prev = key; 346 } 347 return 0; 348 } 349 350 void perf_kwork__report_cleanup_bpf(void) 351 { 352 kwork_trace_bpf__destroy(skel); 353 } 354