1 // SPDX-License-Identifier: GPL-2.0 2 #include "util/bpf_counter.h" 3 #include "util/debug.h" 4 #include "util/evsel.h" 5 #include "util/evlist.h" 6 #include "util/off_cpu.h" 7 #include "util/perf-hooks.h" 8 #include "util/record.h" 9 #include "util/session.h" 10 #include "util/target.h" 11 #include "util/cpumap.h" 12 #include "util/thread_map.h" 13 #include "util/cgroup.h" 14 #include "util/strlist.h" 15 #include <bpf/bpf.h> 16 17 #include "bpf_skel/off_cpu.skel.h" 18 19 #define MAX_STACKS 32 20 #define MAX_PROC 4096 21 /* we don't need actual timestamp, just want to put the samples at last */ 22 #define OFF_CPU_TIMESTAMP (~0ull << 32) 23 24 static struct off_cpu_bpf *skel; 25 26 struct off_cpu_key { 27 u32 pid; 28 u32 tgid; 29 u32 stack_id; 30 u32 state; 31 u64 cgroup_id; 32 }; 33 34 union off_cpu_data { 35 struct perf_event_header hdr; 36 u64 array[1024 / sizeof(u64)]; 37 }; 38 39 static int off_cpu_config(struct evlist *evlist) 40 { 41 struct evsel *evsel; 42 struct perf_event_attr attr = { 43 .type = PERF_TYPE_SOFTWARE, 44 .config = PERF_COUNT_SW_BPF_OUTPUT, 45 .size = sizeof(attr), /* to capture ABI version */ 46 }; 47 char *evname = strdup(OFFCPU_EVENT); 48 49 if (evname == NULL) 50 return -ENOMEM; 51 52 evsel = evsel__new(&attr); 53 if (!evsel) { 54 free(evname); 55 return -ENOMEM; 56 } 57 58 evsel->core.attr.freq = 1; 59 evsel->core.attr.sample_period = 1; 60 /* off-cpu analysis depends on stack trace */ 61 evsel->core.attr.sample_type = PERF_SAMPLE_CALLCHAIN; 62 63 evlist__add(evlist, evsel); 64 65 free(evsel->name); 66 evsel->name = evname; 67 68 return 0; 69 } 70 71 static void off_cpu_start(void *arg) 72 { 73 struct evlist *evlist = arg; 74 75 /* update task filter for the given workload */ 76 if (skel->rodata->has_task && skel->rodata->uses_tgid && 77 perf_thread_map__pid(evlist->core.threads, 0) != -1) { 78 int fd; 79 u32 pid; 80 u8 val = 1; 81 82 fd = bpf_map__fd(skel->maps.task_filter); 83 pid = perf_thread_map__pid(evlist->core.threads, 0); 84 bpf_map_update_elem(fd, &pid, &val, BPF_ANY); 85 } 86 87 skel->bss->enabled = 1; 88 } 89 90 static void off_cpu_finish(void *arg __maybe_unused) 91 { 92 skel->bss->enabled = 0; 93 off_cpu_bpf__destroy(skel); 94 } 95 96 /* v5.18 kernel added prev_state arg, so it needs to check the signature */ 97 static void check_sched_switch_args(void) 98 { 99 struct btf *btf = btf__load_vmlinux_btf(); 100 const struct btf_type *t1, *t2, *t3; 101 u32 type_id; 102 103 if (!btf) { 104 pr_debug("Missing btf, check if CONFIG_DEBUG_INFO_BTF is enabled\n"); 105 goto cleanup; 106 } 107 108 type_id = btf__find_by_name_kind(btf, "btf_trace_sched_switch", 109 BTF_KIND_TYPEDEF); 110 if ((s32)type_id < 0) 111 goto cleanup; 112 113 t1 = btf__type_by_id(btf, type_id); 114 if (t1 == NULL) 115 goto cleanup; 116 117 t2 = btf__type_by_id(btf, t1->type); 118 if (t2 == NULL || !btf_is_ptr(t2)) 119 goto cleanup; 120 121 t3 = btf__type_by_id(btf, t2->type); 122 /* btf_trace func proto has one more argument for the context */ 123 if (t3 && btf_is_func_proto(t3) && btf_vlen(t3) == 5) { 124 /* new format: pass prev_state as 4th arg */ 125 skel->rodata->has_prev_state = true; 126 } 127 cleanup: 128 btf__free(btf); 129 } 130 131 int off_cpu_prepare(struct evlist *evlist, struct target *target, 132 struct record_opts *opts) 133 { 134 int err, fd, i; 135 int ncpus = 1, ntasks = 1, ncgrps = 1; 136 struct strlist *pid_slist = NULL; 137 struct str_node *pos; 138 139 if (off_cpu_config(evlist) < 0) { 140 pr_err("Failed to config off-cpu BPF event\n"); 141 return -1; 142 } 143 144 skel = off_cpu_bpf__open(); 145 if (!skel) { 146 pr_err("Failed to open off-cpu BPF skeleton\n"); 147 return -1; 148 } 149 150 /* don't need to set cpu filter for system-wide mode */ 151 if (target->cpu_list) { 152 ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus); 153 bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); 154 skel->rodata->has_cpu = 1; 155 } 156 157 if (target->pid) { 158 pid_slist = strlist__new(target->pid, NULL); 159 if (!pid_slist) { 160 pr_err("Failed to create a strlist for pid\n"); 161 return -1; 162 } 163 164 ntasks = 0; 165 strlist__for_each_entry(pos, pid_slist) { 166 char *end_ptr; 167 int pid = strtol(pos->s, &end_ptr, 10); 168 169 if (pid == INT_MIN || pid == INT_MAX || 170 (*end_ptr != '\0' && *end_ptr != ',')) 171 continue; 172 173 ntasks++; 174 } 175 176 if (ntasks < MAX_PROC) 177 ntasks = MAX_PROC; 178 179 bpf_map__set_max_entries(skel->maps.task_filter, ntasks); 180 skel->rodata->has_task = 1; 181 skel->rodata->uses_tgid = 1; 182 } else if (target__has_task(target)) { 183 ntasks = perf_thread_map__nr(evlist->core.threads); 184 bpf_map__set_max_entries(skel->maps.task_filter, ntasks); 185 skel->rodata->has_task = 1; 186 } else if (target__none(target)) { 187 bpf_map__set_max_entries(skel->maps.task_filter, MAX_PROC); 188 skel->rodata->has_task = 1; 189 skel->rodata->uses_tgid = 1; 190 } 191 192 if (evlist__first(evlist)->cgrp) { 193 ncgrps = evlist->core.nr_entries - 1; /* excluding a dummy */ 194 bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps); 195 196 if (!cgroup_is_v2("perf_event")) 197 skel->rodata->uses_cgroup_v1 = true; 198 skel->rodata->has_cgroup = 1; 199 } 200 201 if (opts->record_cgroup) { 202 skel->rodata->needs_cgroup = true; 203 204 if (!cgroup_is_v2("perf_event")) 205 skel->rodata->uses_cgroup_v1 = true; 206 } 207 208 set_max_rlimit(); 209 check_sched_switch_args(); 210 211 err = off_cpu_bpf__load(skel); 212 if (err) { 213 pr_err("Failed to load off-cpu skeleton\n"); 214 goto out; 215 } 216 217 if (target->cpu_list) { 218 u32 cpu; 219 u8 val = 1; 220 221 fd = bpf_map__fd(skel->maps.cpu_filter); 222 223 for (i = 0; i < ncpus; i++) { 224 cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu; 225 bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); 226 } 227 } 228 229 if (target->pid) { 230 u8 val = 1; 231 232 fd = bpf_map__fd(skel->maps.task_filter); 233 234 strlist__for_each_entry(pos, pid_slist) { 235 char *end_ptr; 236 u32 tgid; 237 int pid = strtol(pos->s, &end_ptr, 10); 238 239 if (pid == INT_MIN || pid == INT_MAX || 240 (*end_ptr != '\0' && *end_ptr != ',')) 241 continue; 242 243 tgid = pid; 244 bpf_map_update_elem(fd, &tgid, &val, BPF_ANY); 245 } 246 } else if (target__has_task(target)) { 247 u32 pid; 248 u8 val = 1; 249 250 fd = bpf_map__fd(skel->maps.task_filter); 251 252 for (i = 0; i < ntasks; i++) { 253 pid = perf_thread_map__pid(evlist->core.threads, i); 254 bpf_map_update_elem(fd, &pid, &val, BPF_ANY); 255 } 256 } 257 258 if (evlist__first(evlist)->cgrp) { 259 struct evsel *evsel; 260 u8 val = 1; 261 262 fd = bpf_map__fd(skel->maps.cgroup_filter); 263 264 evlist__for_each_entry(evlist, evsel) { 265 struct cgroup *cgrp = evsel->cgrp; 266 267 if (cgrp == NULL) 268 continue; 269 270 if (!cgrp->id && read_cgroup_id(cgrp) < 0) { 271 pr_err("Failed to read cgroup id of %s\n", 272 cgrp->name); 273 goto out; 274 } 275 276 bpf_map_update_elem(fd, &cgrp->id, &val, BPF_ANY); 277 } 278 } 279 280 err = off_cpu_bpf__attach(skel); 281 if (err) { 282 pr_err("Failed to attach off-cpu BPF skeleton\n"); 283 goto out; 284 } 285 286 if (perf_hooks__set_hook("record_start", off_cpu_start, evlist) || 287 perf_hooks__set_hook("record_end", off_cpu_finish, evlist)) { 288 pr_err("Failed to attach off-cpu skeleton\n"); 289 goto out; 290 } 291 292 return 0; 293 294 out: 295 off_cpu_bpf__destroy(skel); 296 return -1; 297 } 298 299 int off_cpu_write(struct perf_session *session) 300 { 301 int bytes = 0, size; 302 int fd, stack; 303 u64 sample_type, val, sid = 0; 304 struct evsel *evsel; 305 struct perf_data_file *file = &session->data->file; 306 struct off_cpu_key prev, key; 307 union off_cpu_data data = { 308 .hdr = { 309 .type = PERF_RECORD_SAMPLE, 310 .misc = PERF_RECORD_MISC_USER, 311 }, 312 }; 313 u64 tstamp = OFF_CPU_TIMESTAMP; 314 315 skel->bss->enabled = 0; 316 317 evsel = evlist__find_evsel_by_str(session->evlist, OFFCPU_EVENT); 318 if (evsel == NULL) { 319 pr_err("%s evsel not found\n", OFFCPU_EVENT); 320 return 0; 321 } 322 323 sample_type = evsel->core.attr.sample_type; 324 325 if (sample_type & ~OFFCPU_SAMPLE_TYPES) { 326 pr_err("not supported sample type: %llx\n", 327 (unsigned long long)sample_type); 328 return -1; 329 } 330 331 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) { 332 if (evsel->core.id) 333 sid = evsel->core.id[0]; 334 } 335 336 fd = bpf_map__fd(skel->maps.off_cpu); 337 stack = bpf_map__fd(skel->maps.stacks); 338 memset(&prev, 0, sizeof(prev)); 339 340 while (!bpf_map_get_next_key(fd, &prev, &key)) { 341 int n = 1; /* start from perf_event_header */ 342 int ip_pos = -1; 343 344 bpf_map_lookup_elem(fd, &key, &val); 345 346 if (sample_type & PERF_SAMPLE_IDENTIFIER) 347 data.array[n++] = sid; 348 if (sample_type & PERF_SAMPLE_IP) { 349 ip_pos = n; 350 data.array[n++] = 0; /* will be updated */ 351 } 352 if (sample_type & PERF_SAMPLE_TID) 353 data.array[n++] = (u64)key.pid << 32 | key.tgid; 354 if (sample_type & PERF_SAMPLE_TIME) 355 data.array[n++] = tstamp; 356 if (sample_type & PERF_SAMPLE_ID) 357 data.array[n++] = sid; 358 if (sample_type & PERF_SAMPLE_CPU) 359 data.array[n++] = 0; 360 if (sample_type & PERF_SAMPLE_PERIOD) 361 data.array[n++] = val; 362 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 363 int len = 0; 364 365 /* data.array[n] is callchain->nr (updated later) */ 366 data.array[n + 1] = PERF_CONTEXT_USER; 367 data.array[n + 2] = 0; 368 369 bpf_map_lookup_elem(stack, &key.stack_id, &data.array[n + 2]); 370 while (data.array[n + 2 + len]) 371 len++; 372 373 /* update length of callchain */ 374 data.array[n] = len + 1; 375 376 /* update sample ip with the first callchain entry */ 377 if (ip_pos >= 0) 378 data.array[ip_pos] = data.array[n + 2]; 379 380 /* calculate sample callchain data array length */ 381 n += len + 2; 382 } 383 if (sample_type & PERF_SAMPLE_CGROUP) 384 data.array[n++] = key.cgroup_id; 385 386 size = n * sizeof(u64); 387 data.hdr.size = size; 388 bytes += size; 389 390 if (perf_data_file__write(file, &data, size) < 0) { 391 pr_err("failed to write perf data, error: %m\n"); 392 return bytes; 393 } 394 395 prev = key; 396 /* increase dummy timestamp to sort later samples */ 397 tstamp++; 398 } 399 return bytes; 400 } 401