1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/time64.h> 3 4 #include <bpf/bpf.h> 5 #include <bpf/btf.h> 6 #include <internal/xyarray.h> 7 8 #include "bpf_counter.h" 9 #include "bpf_skel/off_cpu.skel.h" 10 #include "cgroup.h" 11 #include "cpumap.h" 12 #include "debug.h" 13 #include "evlist.h" 14 #include "evsel.h" 15 #include "off_cpu.h" 16 #include "parse-events.h" 17 #include "perf-hooks.h" 18 #include "record.h" 19 #include "session.h" 20 #include "strlist.h" 21 #include "target.h" 22 #include "thread_map.h" 23 24 #define MAX_STACKS 32 25 #define MAX_PROC 4096 26 /* we don't need actual timestamp, just want to put the samples at last */ 27 #define OFF_CPU_TIMESTAMP (~0ull << 32) 28 29 static struct off_cpu_bpf *skel; 30 31 struct off_cpu_key { 32 u32 pid; 33 u32 tgid; 34 u32 stack_id; 35 u32 state; 36 u64 cgroup_id; 37 }; 38 39 union off_cpu_data { 40 struct perf_event_header hdr; 41 u64 array[1024 / sizeof(u64)]; 42 }; 43 44 static u64 off_cpu_raw[MAX_STACKS + 5]; 45 46 static int off_cpu_config(struct evlist *evlist) 47 { 48 char off_cpu_event[64]; 49 struct evsel *evsel; 50 51 scnprintf(off_cpu_event, sizeof(off_cpu_event), "bpf-output/name=%s/", OFFCPU_EVENT); 52 if (parse_event(evlist, off_cpu_event)) { 53 pr_err("Failed to open off-cpu event\n"); 54 return -1; 55 } 56 57 evlist__for_each_entry(evlist, evsel) { 58 if (evsel__is_offcpu_event(evsel)) { 59 evsel->core.system_wide = true; 60 break; 61 } 62 } 63 64 return 0; 65 } 66 67 static void off_cpu_start(void *arg) 68 { 69 struct evlist *evlist = arg; 70 struct evsel *evsel; 71 struct perf_cpu pcpu; 72 unsigned int i; 73 74 /* update task filter for the given workload */ 75 if (skel->rodata->has_task && skel->rodata->uses_tgid && 76 perf_thread_map__pid(evlist__core(evlist)->threads, 0) != -1) { 77 int fd; 78 u32 pid; 79 u8 val = 1; 80 81 fd = bpf_map__fd(skel->maps.task_filter); 82 pid = perf_thread_map__pid(evlist__core(evlist)->threads, 0); 83 bpf_map_update_elem(fd, &pid, &val, BPF_ANY); 84 } 85 86 /* update BPF perf_event map */ 87 evsel = evlist__find_evsel_by_str(evlist, OFFCPU_EVENT); 88 if (evsel == NULL) { 89 pr_err("%s evsel not found\n", OFFCPU_EVENT); 90 return; 91 } 92 93 perf_cpu_map__for_each_cpu(pcpu, i, evsel->core.cpus) { 94 int err; 95 int cpu_nr = pcpu.cpu; 96 97 err = bpf_map__update_elem(skel->maps.offcpu_output, &cpu_nr, sizeof(int), 98 xyarray__entry(evsel->core.fd, cpu_nr, 0), 99 sizeof(int), BPF_ANY); 100 if (err) { 101 pr_err("Failed to update perf event map for direct off-cpu dumping\n"); 102 return; 103 } 104 } 105 106 skel->bss->enabled = 1; 107 } 108 109 static void off_cpu_finish(void *arg __maybe_unused) 110 { 111 skel->bss->enabled = 0; 112 off_cpu_bpf__destroy(skel); 113 } 114 115 /* v5.18 kernel added prev_state arg, so it needs to check the signature */ 116 static void check_sched_switch_args(void) 117 { 118 struct btf *btf = btf__load_vmlinux_btf(); 119 const struct btf_type *t1, *t2, *t3; 120 u32 type_id; 121 122 if (!btf) { 123 pr_debug("Missing btf, check if CONFIG_DEBUG_INFO_BTF is enabled\n"); 124 goto cleanup; 125 } 126 127 type_id = btf__find_by_name_kind(btf, "btf_trace_sched_switch", 128 BTF_KIND_TYPEDEF); 129 if ((s32)type_id < 0) 130 goto cleanup; 131 132 t1 = btf__type_by_id(btf, type_id); 133 if (t1 == NULL) 134 goto cleanup; 135 136 t2 = btf__type_by_id(btf, t1->type); 137 if (t2 == NULL || !btf_is_ptr(t2)) 138 goto cleanup; 139 140 t3 = btf__type_by_id(btf, t2->type); 141 /* btf_trace func proto has one more argument for the context */ 142 if (t3 && btf_is_func_proto(t3) && btf_vlen(t3) == 5) { 143 /* new format: pass prev_state as 4th arg */ 144 skel->rodata->has_prev_state = true; 145 } 146 cleanup: 147 btf__free(btf); 148 } 149 150 int off_cpu_prepare(struct evlist *evlist, struct target *target, 151 struct record_opts *opts) 152 { 153 int err, fd, i; 154 int ncpus = 1, ntasks = 1, ncgrps = 1; 155 struct strlist *pid_slist = NULL; 156 struct str_node *pos; 157 158 if (off_cpu_config(evlist) < 0) { 159 pr_err("Failed to config off-cpu BPF event\n"); 160 return -1; 161 } 162 163 skel = off_cpu_bpf__open(); 164 if (!skel) { 165 pr_err("Failed to open off-cpu BPF skeleton\n"); 166 return -1; 167 } 168 169 /* don't need to set cpu filter for system-wide mode */ 170 if (target->cpu_list) { 171 ncpus = perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus); 172 bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); 173 skel->rodata->has_cpu = 1; 174 } 175 176 if (target->pid) { 177 pid_slist = strlist__new(target->pid, NULL); 178 if (!pid_slist) { 179 pr_err("Failed to create a strlist for pid\n"); 180 return -1; 181 } 182 183 ntasks = 0; 184 strlist__for_each_entry(pos, pid_slist) { 185 char *end_ptr; 186 int pid = strtol(pos->s, &end_ptr, 10); 187 188 if (pid == INT_MIN || pid == INT_MAX || 189 (*end_ptr != '\0' && *end_ptr != ',')) 190 continue; 191 192 ntasks++; 193 } 194 195 if (ntasks < MAX_PROC) 196 ntasks = MAX_PROC; 197 198 bpf_map__set_max_entries(skel->maps.task_filter, ntasks); 199 skel->rodata->has_task = 1; 200 skel->rodata->uses_tgid = 1; 201 } else if (target__has_task(target)) { 202 ntasks = perf_thread_map__nr(evlist__core(evlist)->threads); 203 bpf_map__set_max_entries(skel->maps.task_filter, ntasks); 204 skel->rodata->has_task = 1; 205 } else if (target__none(target)) { 206 bpf_map__set_max_entries(skel->maps.task_filter, MAX_PROC); 207 skel->rodata->has_task = 1; 208 skel->rodata->uses_tgid = 1; 209 } 210 211 if (evlist__first(evlist)->cgrp) { 212 ncgrps = evlist__nr_entries(evlist) - 1; /* excluding a dummy */ 213 bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps); 214 215 if (!cgroup_is_v2("perf_event")) 216 skel->rodata->uses_cgroup_v1 = true; 217 skel->rodata->has_cgroup = 1; 218 } 219 220 if (opts->record_cgroup) { 221 skel->rodata->needs_cgroup = true; 222 223 if (!cgroup_is_v2("perf_event")) 224 skel->rodata->uses_cgroup_v1 = true; 225 } 226 227 set_max_rlimit(); 228 check_sched_switch_args(); 229 230 err = off_cpu_bpf__load(skel); 231 if (err) { 232 pr_err("Failed to load off-cpu skeleton\n"); 233 goto out; 234 } 235 236 if (target->cpu_list) { 237 u32 cpu; 238 u8 val = 1; 239 240 fd = bpf_map__fd(skel->maps.cpu_filter); 241 242 for (i = 0; i < ncpus; i++) { 243 cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, i).cpu; 244 bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); 245 } 246 } 247 248 if (target->pid) { 249 u8 val = 1; 250 251 fd = bpf_map__fd(skel->maps.task_filter); 252 253 strlist__for_each_entry(pos, pid_slist) { 254 char *end_ptr; 255 u32 tgid; 256 int pid = strtol(pos->s, &end_ptr, 10); 257 258 if (pid == INT_MIN || pid == INT_MAX || 259 (*end_ptr != '\0' && *end_ptr != ',')) 260 continue; 261 262 tgid = pid; 263 bpf_map_update_elem(fd, &tgid, &val, BPF_ANY); 264 } 265 } else if (target__has_task(target)) { 266 u32 pid; 267 u8 val = 1; 268 269 fd = bpf_map__fd(skel->maps.task_filter); 270 271 for (i = 0; i < ntasks; i++) { 272 pid = perf_thread_map__pid(evlist__core(evlist)->threads, i); 273 bpf_map_update_elem(fd, &pid, &val, BPF_ANY); 274 } 275 } 276 277 if (evlist__first(evlist)->cgrp) { 278 struct evsel *evsel; 279 u8 val = 1; 280 281 fd = bpf_map__fd(skel->maps.cgroup_filter); 282 283 evlist__for_each_entry(evlist, evsel) { 284 struct cgroup *cgrp = evsel->cgrp; 285 286 if (cgrp == NULL) 287 continue; 288 289 if (!cgrp->id && read_cgroup_id(cgrp) < 0) { 290 pr_err("Failed to read cgroup id of %s\n", 291 cgrp->name); 292 goto out; 293 } 294 295 bpf_map_update_elem(fd, &cgrp->id, &val, BPF_ANY); 296 } 297 } 298 299 skel->bss->offcpu_thresh_ns = opts->off_cpu_thresh_ns; 300 301 err = off_cpu_bpf__attach(skel); 302 if (err) { 303 pr_err("Failed to attach off-cpu BPF skeleton\n"); 304 goto out; 305 } 306 307 if (perf_hooks__set_hook("record_start", off_cpu_start, evlist) || 308 perf_hooks__set_hook("record_end", off_cpu_finish, evlist)) { 309 pr_err("Failed to attach off-cpu skeleton\n"); 310 goto out; 311 } 312 313 return 0; 314 315 out: 316 off_cpu_bpf__destroy(skel); 317 return -1; 318 } 319 320 int off_cpu_write(struct perf_session *session) 321 { 322 int bytes = 0, size; 323 int fd, stack; 324 u32 raw_size; 325 u64 sample_type, val, sid = 0; 326 struct evsel *evsel; 327 struct perf_data_file *file = &session->data->file; 328 struct off_cpu_key prev, key; 329 union off_cpu_data data = { 330 .hdr = { 331 .type = PERF_RECORD_SAMPLE, 332 .misc = PERF_RECORD_MISC_USER, 333 }, 334 }; 335 u64 tstamp = OFF_CPU_TIMESTAMP; 336 337 skel->bss->enabled = 0; 338 339 evsel = evlist__find_evsel_by_str(session->evlist, OFFCPU_EVENT); 340 if (evsel == NULL) { 341 pr_err("%s evsel not found\n", OFFCPU_EVENT); 342 return 0; 343 } 344 345 sample_type = evsel->core.attr.sample_type; 346 347 if (sample_type & ~OFFCPU_SAMPLE_TYPES) { 348 pr_err("not supported sample type: %llx\n", 349 (unsigned long long)sample_type); 350 return -1; 351 } 352 353 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) { 354 if (evsel->core.id) 355 sid = evsel->core.id[0]; 356 } 357 358 fd = bpf_map__fd(skel->maps.off_cpu); 359 stack = bpf_map__fd(skel->maps.stacks); 360 memset(&prev, 0, sizeof(prev)); 361 362 while (!bpf_map_get_next_key(fd, &prev, &key)) { 363 int n = 1; /* start from perf_event_header */ 364 365 bpf_map_lookup_elem(fd, &key, &val); 366 367 /* zero-fill some of the fields, will be overwritten by raw_data when parsing */ 368 if (sample_type & PERF_SAMPLE_IDENTIFIER) 369 data.array[n++] = sid; 370 if (sample_type & PERF_SAMPLE_IP) 371 data.array[n++] = 0; /* will be updated */ 372 if (sample_type & PERF_SAMPLE_TID) 373 data.array[n++] = 0; 374 if (sample_type & PERF_SAMPLE_TIME) 375 data.array[n++] = tstamp; 376 if (sample_type & PERF_SAMPLE_CPU) 377 data.array[n++] = 0; 378 if (sample_type & PERF_SAMPLE_PERIOD) 379 data.array[n++] = 0; 380 if (sample_type & PERF_SAMPLE_RAW) { 381 /* 382 * [ size ][ data ] 383 * [ data ] 384 * [ data ] 385 * [ data ] 386 * [ data ][ empty] 387 */ 388 int len = 0, i = 0; 389 void *raw_data = (void *)data.array + n * sizeof(u64); 390 391 off_cpu_raw[i++] = (u64)key.pid << 32 | key.tgid; 392 off_cpu_raw[i++] = val; 393 394 /* off_cpu_raw[i] is callchain->nr (updated later) */ 395 off_cpu_raw[i + 1] = PERF_CONTEXT_USER; 396 off_cpu_raw[i + 2] = 0; 397 398 bpf_map_lookup_elem(stack, &key.stack_id, &off_cpu_raw[i + 2]); 399 while (off_cpu_raw[i + 2 + len]) 400 len++; 401 402 off_cpu_raw[i] = len + 1; 403 i += len + 2; 404 405 off_cpu_raw[i++] = key.cgroup_id; 406 407 raw_size = i * sizeof(u64) + sizeof(u32); /* 4 bytes for alignment */ 408 memcpy(raw_data, &raw_size, sizeof(raw_size)); 409 memcpy(raw_data + sizeof(u32), off_cpu_raw, i * sizeof(u64)); 410 411 n += i + 1; 412 } 413 if (sample_type & PERF_SAMPLE_CGROUP) 414 data.array[n++] = key.cgroup_id; 415 416 size = n * sizeof(u64); 417 data.hdr.size = size; 418 bytes += size; 419 420 if (perf_data_file__write(file, &data, size) < 0) { 421 pr_err("failed to write perf data, error: %m\n"); 422 return bytes; 423 } 424 425 prev = key; 426 /* increase dummy timestamp to sort later samples */ 427 tstamp++; 428 } 429 return bytes; 430 } 431