1 // SPDX-License-Identifier: GPL-2.0 2 #include <errno.h> 3 #include <stddef.h> 4 #include <stdint.h> 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <bpf/bpf.h> 9 #include <bpf/btf.h> 10 #include <bpf/libbpf.h> 11 #include <linux/bpf.h> 12 #include <linux/btf.h> 13 #include <linux/err.h> 14 #include <linux/perf_event.h> 15 #include <linux/string.h> 16 #include <linux/zalloc.h> 17 #include <internal/lib.h> 18 #include <perf/event.h> 19 #include <symbol/kallsyms.h> 20 #include "bpf-event.h" 21 #include "bpf-utils.h" 22 #include "debug.h" 23 #include "dso.h" 24 #include "symbol.h" 25 #include "machine.h" 26 #include "env.h" 27 #include "session.h" 28 #include "map.h" 29 #include "evlist.h" 30 #include "record.h" 31 #include "util/synthetic-events.h" 32 33 static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) 34 { 35 int ret = 0; 36 size_t i; 37 38 for (i = 0; i < len; i++) 39 ret += scnprintf(buf + ret, size - ret, "%02x", data[i]); 40 return ret; 41 } 42 43 static int machine__process_bpf_event_load(struct machine *machine, 44 union perf_event *event, 45 struct perf_sample *sample __maybe_unused) 46 { 47 struct bpf_prog_info_node *info_node; 48 struct perf_env *env = machine->env; 49 struct perf_bpil *info_linear; 50 int id = event->bpf.id; 51 unsigned int i; 52 53 /* perf-record, no need to handle bpf-event */ 54 if (env == NULL) 55 return 0; 56 57 info_node = perf_env__find_bpf_prog_info(env, id); 58 if (!info_node) 59 return 0; 60 info_linear = info_node->info_linear; 61 62 /* jited_ksyms is only valid if bpil_offs_to_addr() converted it */ 63 if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_KSYMS))) 64 return 0; 65 66 for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { 67 u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); 68 u64 addr = addrs[i]; 69 struct map *map = maps__find(machine__kernel_maps(machine), addr); 70 71 if (map) { 72 struct dso *dso = map__dso(map); 73 74 dso__set_binary_type(dso, DSO_BINARY_TYPE__BPF_PROG_INFO); 75 dso__bpf_prog(dso)->id = id; 76 dso__bpf_prog(dso)->sub_id = i; 77 dso__bpf_prog(dso)->env = env; 78 map__put(map); 79 } 80 } 81 return 0; 82 } 83 84 int machine__process_bpf(struct machine *machine, union perf_event *event, 85 struct perf_sample *sample) 86 { 87 if (dump_trace) 88 perf_event__fprintf_bpf(event, stdout); 89 90 switch (event->bpf.type) { 91 case PERF_BPF_EVENT_PROG_LOAD: 92 return machine__process_bpf_event_load(machine, event, sample); 93 94 case PERF_BPF_EVENT_PROG_UNLOAD: 95 /* 96 * Do not free bpf_prog_info and btf of the program here, 97 * as annotation still need them. They will be freed at 98 * the end of the session. 99 */ 100 break; 101 default: 102 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 103 break; 104 } 105 return 0; 106 } 107 108 static int perf_env__fetch_btf(struct perf_env *env, 109 u32 btf_id, 110 struct btf *btf) 111 { 112 struct btf_node *node; 113 u32 data_size; 114 const void *data; 115 116 data = btf__raw_data(btf, &data_size); 117 118 node = malloc(data_size + sizeof(struct btf_node)); 119 if (!node) 120 return -1; 121 122 node->id = btf_id; 123 node->data_size = data_size; 124 memcpy(node->data, data, data_size); 125 126 if (!perf_env__insert_btf(env, node)) { 127 /* Insertion failed because of a duplicate. */ 128 free(node); 129 return -1; 130 } 131 return 0; 132 } 133 134 static int synthesize_bpf_prog_name(char *buf, int size, 135 struct bpf_prog_info *info, 136 struct btf *btf, 137 u32 sub_id) 138 { 139 u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags); 140 void *func_infos = (void *)(uintptr_t)(info->func_info); 141 u32 sub_prog_cnt = info->nr_jited_ksyms; 142 const struct bpf_func_info *finfo; 143 const char *short_name = NULL; 144 const struct btf_type *t; 145 int name_len; 146 147 name_len = scnprintf(buf, size, "bpf_prog_"); 148 name_len += snprintf_hex(buf + name_len, size - name_len, 149 prog_tags[sub_id], BPF_TAG_SIZE); 150 if (btf && 151 info->func_info_rec_size >= sizeof(*finfo) && 152 sub_id < info->nr_func_info) { 153 finfo = func_infos + sub_id * info->func_info_rec_size; 154 t = btf__type_by_id(btf, finfo->type_id); 155 if (t) 156 short_name = btf__name_by_offset(btf, t->name_off); 157 } else if (sub_id == 0 && sub_prog_cnt == 1) { 158 /* no subprog */ 159 if (info->name[0]) 160 short_name = info->name; 161 } else 162 short_name = "F"; 163 if (short_name) { 164 name_len += scnprintf(buf + name_len, size - name_len, 165 "_%s", short_name); 166 } 167 return name_len; 168 } 169 170 #ifdef HAVE_LIBBPF_STRINGS_SUPPORT 171 172 #define BPF_METADATA_PREFIX "bpf_metadata_" 173 #define BPF_METADATA_PREFIX_LEN (sizeof(BPF_METADATA_PREFIX) - 1) 174 175 static bool name_has_bpf_metadata_prefix(const char **s) 176 { 177 if (strncmp(*s, BPF_METADATA_PREFIX, BPF_METADATA_PREFIX_LEN) != 0) 178 return false; 179 *s += BPF_METADATA_PREFIX_LEN; 180 return true; 181 } 182 183 struct bpf_metadata_map { 184 struct btf *btf; 185 const struct btf_type *datasec; 186 void *rodata; 187 size_t rodata_size; 188 unsigned int num_vars; 189 }; 190 191 static int bpf_metadata_read_map_data(__u32 map_id, struct bpf_metadata_map *map) 192 { 193 int map_fd; 194 struct bpf_map_info map_info; 195 __u32 map_info_len; 196 int key; 197 struct btf *btf; 198 const struct btf_type *datasec; 199 struct btf_var_secinfo *vsi; 200 unsigned int vlen, vars; 201 void *rodata; 202 203 map_fd = bpf_map_get_fd_by_id(map_id); 204 if (map_fd < 0) 205 return -1; 206 207 memset(&map_info, 0, sizeof(map_info)); 208 map_info_len = sizeof(map_info); 209 if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len) < 0) 210 goto out_close; 211 212 /* If it's not an .rodata map, don't bother. */ 213 if (map_info.type != BPF_MAP_TYPE_ARRAY || 214 map_info.key_size != sizeof(int) || 215 map_info.max_entries != 1 || 216 !map_info.btf_value_type_id || 217 !strstr(map_info.name, ".rodata")) { 218 goto out_close; 219 } 220 221 btf = btf__load_from_kernel_by_id(map_info.btf_id); 222 if (!btf) 223 goto out_close; 224 datasec = btf__type_by_id(btf, map_info.btf_value_type_id); 225 if (!btf_is_datasec(datasec)) 226 goto out_free_btf; 227 228 /* 229 * If there aren't any variables with the "bpf_metadata_" prefix, 230 * don't bother. 231 */ 232 vlen = btf_vlen(datasec); 233 vsi = btf_var_secinfos(datasec); 234 vars = 0; 235 for (unsigned int i = 0; i < vlen; i++, vsi++) { 236 const struct btf_type *t_var = btf__type_by_id(btf, vsi->type); 237 const char *name = btf__name_by_offset(btf, t_var->name_off); 238 239 if (name_has_bpf_metadata_prefix(&name)) 240 vars++; 241 } 242 if (vars == 0) 243 goto out_free_btf; 244 245 rodata = zalloc(map_info.value_size); 246 if (!rodata) 247 goto out_free_btf; 248 key = 0; 249 if (bpf_map_lookup_elem(map_fd, &key, rodata)) { 250 free(rodata); 251 goto out_free_btf; 252 } 253 close(map_fd); 254 255 map->btf = btf; 256 map->datasec = datasec; 257 map->rodata = rodata; 258 map->rodata_size = map_info.value_size; 259 map->num_vars = vars; 260 return 0; 261 262 out_free_btf: 263 btf__free(btf); 264 out_close: 265 close(map_fd); 266 return -1; 267 } 268 269 struct format_btf_ctx { 270 char *buf; 271 size_t buf_size; 272 size_t buf_idx; 273 }; 274 275 static void format_btf_cb(void *arg, const char *fmt, va_list ap) 276 { 277 int n; 278 struct format_btf_ctx *ctx = (struct format_btf_ctx *)arg; 279 280 n = vsnprintf(ctx->buf + ctx->buf_idx, ctx->buf_size - ctx->buf_idx, 281 fmt, ap); 282 ctx->buf_idx += n; 283 if (ctx->buf_idx >= ctx->buf_size) 284 ctx->buf_idx = ctx->buf_size; 285 } 286 287 static void format_btf_variable(struct btf *btf, char *buf, size_t buf_size, 288 const struct btf_type *t, const void *btf_data) 289 { 290 struct format_btf_ctx ctx = { 291 .buf = buf, 292 .buf_idx = 0, 293 .buf_size = buf_size, 294 }; 295 const struct btf_dump_type_data_opts opts = { 296 .sz = sizeof(struct btf_dump_type_data_opts), 297 .skip_names = 1, 298 .compact = 1, 299 .emit_strings = 1, 300 }; 301 struct btf_dump *d; 302 size_t btf_size; 303 304 d = btf_dump__new(btf, format_btf_cb, &ctx, NULL); 305 btf_size = btf__resolve_size(btf, t->type); 306 btf_dump__dump_type_data(d, t->type, btf_data, btf_size, &opts); 307 btf_dump__free(d); 308 } 309 310 static void bpf_metadata_fill_event(struct bpf_metadata_map *map, 311 struct perf_record_bpf_metadata *bpf_metadata_event) 312 { 313 struct btf_var_secinfo *vsi; 314 unsigned int i, vlen; 315 316 memset(bpf_metadata_event->prog_name, 0, BPF_PROG_NAME_LEN); 317 vlen = btf_vlen(map->datasec); 318 vsi = btf_var_secinfos(map->datasec); 319 320 for (i = 0; i < vlen; i++, vsi++) { 321 const struct btf_type *t_var = btf__type_by_id(map->btf, 322 vsi->type); 323 const char *name = btf__name_by_offset(map->btf, 324 t_var->name_off); 325 const __u64 nr_entries = bpf_metadata_event->nr_entries; 326 struct perf_record_bpf_metadata_entry *entry; 327 328 if (!name_has_bpf_metadata_prefix(&name)) 329 continue; 330 331 if (nr_entries >= (__u64)map->num_vars) 332 break; 333 334 entry = &bpf_metadata_event->entries[nr_entries]; 335 memset(entry, 0, sizeof(*entry)); 336 snprintf(entry->key, BPF_METADATA_KEY_LEN, "%s", name); 337 format_btf_variable(map->btf, entry->value, 338 BPF_METADATA_VALUE_LEN, t_var, 339 map->rodata + vsi->offset); 340 bpf_metadata_event->nr_entries++; 341 } 342 } 343 344 static void bpf_metadata_free_map_data(struct bpf_metadata_map *map) 345 { 346 btf__free(map->btf); 347 free(map->rodata); 348 } 349 350 static struct bpf_metadata *bpf_metadata_alloc(__u32 nr_prog_tags, 351 __u32 nr_variables) 352 { 353 struct bpf_metadata *metadata; 354 size_t event_size; 355 356 metadata = zalloc(sizeof(struct bpf_metadata)); 357 if (!metadata) 358 return NULL; 359 360 metadata->prog_names = calloc(nr_prog_tags, sizeof(char *)); 361 if (!metadata->prog_names) { 362 bpf_metadata_free(metadata); 363 return NULL; 364 } 365 for (__u32 prog_index = 0; prog_index < nr_prog_tags; prog_index++) { 366 metadata->prog_names[prog_index] = zalloc(BPF_PROG_NAME_LEN); 367 if (!metadata->prog_names[prog_index]) { 368 bpf_metadata_free(metadata); 369 return NULL; 370 } 371 metadata->nr_prog_names++; 372 } 373 374 event_size = sizeof(metadata->event->bpf_metadata) + 375 nr_variables * sizeof(metadata->event->bpf_metadata.entries[0]); 376 /* 377 * header.size is __u16. synthesize_perf_record_bpf_metadata() 378 * adds machine->id_hdr_size (up to ~64 bytes) after this, so 379 * leave headroom to prevent the final size from wrapping. 380 */ 381 if (event_size > UINT16_MAX - 256) { 382 bpf_metadata_free(metadata); 383 return NULL; 384 } 385 metadata->event = zalloc(event_size); 386 if (!metadata->event) { 387 bpf_metadata_free(metadata); 388 return NULL; 389 } 390 metadata->event->bpf_metadata = (struct perf_record_bpf_metadata) { 391 .header = { 392 .type = PERF_RECORD_BPF_METADATA, 393 .size = event_size, 394 }, 395 .nr_entries = 0, 396 }; 397 398 return metadata; 399 } 400 401 static struct bpf_metadata *bpf_metadata_create(struct bpf_prog_info *info) 402 { 403 struct bpf_metadata *metadata; 404 const __u32 *map_ids = (__u32 *)(uintptr_t)info->map_ids; 405 406 for (__u32 map_index = 0; map_index < info->nr_map_ids; map_index++) { 407 struct bpf_metadata_map map; 408 409 if (bpf_metadata_read_map_data(map_ids[map_index], &map) != 0) 410 continue; 411 412 metadata = bpf_metadata_alloc(info->nr_prog_tags, map.num_vars); 413 if (!metadata) { 414 bpf_metadata_free_map_data(&map); 415 continue; 416 } 417 418 bpf_metadata_fill_event(&map, &metadata->event->bpf_metadata); 419 420 for (__u32 index = 0; index < info->nr_prog_tags; index++) { 421 synthesize_bpf_prog_name(metadata->prog_names[index], 422 BPF_PROG_NAME_LEN, info, 423 map.btf, index); 424 } 425 426 bpf_metadata_free_map_data(&map); 427 428 return metadata; 429 } 430 431 return NULL; 432 } 433 434 static int synthesize_perf_record_bpf_metadata(const struct bpf_metadata *metadata, 435 const struct perf_tool *tool, 436 perf_event__handler_t process, 437 struct machine *machine) 438 { 439 const size_t event_size = metadata->event->header.size; 440 union perf_event *event; 441 int err = 0; 442 443 event = zalloc(event_size + machine->id_hdr_size); 444 if (!event) 445 return -1; 446 memcpy(event, metadata->event, event_size); 447 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 448 event->header.size += machine->id_hdr_size; 449 for (__u32 index = 0; index < metadata->nr_prog_names; index++) { 450 memcpy(event->bpf_metadata.prog_name, 451 metadata->prog_names[index], BPF_PROG_NAME_LEN); 452 err = perf_tool__process_synth_event(tool, event, machine, 453 process); 454 if (err != 0) 455 break; 456 } 457 458 free(event); 459 return err; 460 } 461 462 void bpf_metadata_free(struct bpf_metadata *metadata) 463 { 464 if (metadata == NULL) 465 return; 466 for (__u32 index = 0; index < metadata->nr_prog_names; index++) 467 free(metadata->prog_names[index]); 468 free(metadata->prog_names); 469 free(metadata->event); 470 free(metadata); 471 } 472 473 #else /* HAVE_LIBBPF_STRINGS_SUPPORT */ 474 475 static struct bpf_metadata *bpf_metadata_create(struct bpf_prog_info *info __maybe_unused) 476 { 477 return NULL; 478 } 479 480 static int synthesize_perf_record_bpf_metadata(const struct bpf_metadata *metadata __maybe_unused, 481 const struct perf_tool *tool __maybe_unused, 482 perf_event__handler_t process __maybe_unused, 483 struct machine *machine __maybe_unused) 484 { 485 return 0; 486 } 487 488 void bpf_metadata_free(struct bpf_metadata *metadata __maybe_unused) 489 { 490 } 491 492 #endif /* HAVE_LIBBPF_STRINGS_SUPPORT */ 493 494 struct bpf_metadata_final_ctx { 495 const struct perf_tool *tool; 496 perf_event__handler_t process; 497 struct machine *machine; 498 }; 499 500 static void synthesize_final_bpf_metadata_cb(struct bpf_prog_info_node *node, 501 void *data) 502 { 503 struct bpf_metadata_final_ctx *ctx = (struct bpf_metadata_final_ctx *)data; 504 struct bpf_metadata *metadata = node->metadata; 505 int err; 506 507 if (metadata == NULL) 508 return; 509 err = synthesize_perf_record_bpf_metadata(metadata, ctx->tool, 510 ctx->process, ctx->machine); 511 if (err != 0) { 512 const char *prog_name = metadata->prog_names[0]; 513 514 if (prog_name != NULL) 515 pr_warning("Couldn't synthesize final BPF metadata for %s.\n", prog_name); 516 else 517 pr_warning("Couldn't synthesize final BPF metadata.\n"); 518 } 519 bpf_metadata_free(metadata); 520 node->metadata = NULL; 521 } 522 523 void perf_event__synthesize_final_bpf_metadata(struct perf_session *session, 524 perf_event__handler_t process) 525 { 526 struct perf_env *env = &session->header.env; 527 struct bpf_metadata_final_ctx ctx = { 528 .tool = session->tool, 529 .process = process, 530 .machine = &session->machines.host, 531 }; 532 533 perf_env__iterate_bpf_prog_info(env, synthesize_final_bpf_metadata_cb, 534 &ctx); 535 } 536 537 /* 538 * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf 539 * program. One PERF_RECORD_BPF_EVENT is generated for the program. And 540 * one PERF_RECORD_KSYMBOL is generated for each sub program. 541 * 542 * Returns: 543 * 0 for success; 544 * -1 for failures; 545 * -2 for lack of kernel support. 546 */ 547 static int perf_event__synthesize_one_bpf_prog(struct perf_session *session, 548 perf_event__handler_t process, 549 struct machine *machine, 550 int fd, 551 union perf_event *event, 552 struct record_opts *opts) 553 { 554 struct perf_record_ksymbol *ksymbol_event = &event->ksymbol; 555 struct perf_record_bpf_event *bpf_event = &event->bpf; 556 const struct perf_tool *tool = session->tool; 557 struct bpf_prog_info_node *info_node; 558 struct perf_bpil *info_linear; 559 struct bpf_metadata *metadata; 560 struct bpf_prog_info *info; 561 struct btf *btf = NULL; 562 struct perf_env *env; 563 u32 sub_prog_cnt, i; 564 int err = 0; 565 u64 arrays; 566 567 /* 568 * for perf-record and perf-report use header.env; 569 * otherwise, use global perf_env. 570 */ 571 env = perf_session__env(session); 572 573 arrays = 1UL << PERF_BPIL_JITED_KSYMS; 574 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; 575 arrays |= 1UL << PERF_BPIL_FUNC_INFO; 576 arrays |= 1UL << PERF_BPIL_PROG_TAGS; 577 arrays |= 1UL << PERF_BPIL_JITED_INSNS; 578 arrays |= 1UL << PERF_BPIL_LINE_INFO; 579 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; 580 arrays |= 1UL << PERF_BPIL_MAP_IDS; 581 582 info_linear = get_bpf_prog_info_linear(fd, arrays); 583 if (IS_ERR_OR_NULL(info_linear)) { 584 info_linear = NULL; 585 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 586 return -1; 587 } 588 589 if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) { 590 free(info_linear); 591 pr_debug("%s: the kernel is too old, aborting\n", __func__); 592 return -2; 593 } 594 595 info = &info_linear->info; 596 if (!info->jited_ksyms) { 597 free(info_linear); 598 return -1; 599 } 600 601 /* number of ksyms, func_lengths, and tags should match */ 602 sub_prog_cnt = info->nr_jited_ksyms; 603 if (sub_prog_cnt != info->nr_prog_tags || 604 sub_prog_cnt != info->nr_jited_func_lens) { 605 free(info_linear); 606 return -1; 607 } 608 609 /* check BTF func info support */ 610 if (info->btf_id && info->nr_func_info && info->func_info_rec_size) { 611 /* btf func info number should be same as sub_prog_cnt */ 612 if (sub_prog_cnt != info->nr_func_info) { 613 pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__); 614 free(info_linear); 615 return -1; 616 } 617 btf = btf__load_from_kernel_by_id(info->btf_id); 618 if (libbpf_get_error(btf)) { 619 pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id); 620 err = -1; 621 goto out; 622 } 623 perf_env__fetch_btf(env, info->btf_id, btf); 624 } 625 626 /* Synthesize PERF_RECORD_KSYMBOL */ 627 for (i = 0; i < sub_prog_cnt; i++) { 628 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 629 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 630 int name_len; 631 632 *ksymbol_event = (struct perf_record_ksymbol) { 633 .header = { 634 .type = PERF_RECORD_KSYMBOL, 635 .size = offsetof(struct perf_record_ksymbol, name), 636 }, 637 .addr = prog_addrs[i], 638 .len = prog_lens[i], 639 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 640 .flags = 0, 641 }; 642 643 name_len = synthesize_bpf_prog_name(ksymbol_event->name, 644 KSYM_NAME_LEN, info, btf, i); 645 ksymbol_event->header.size += PERF_ALIGN(name_len + 1, 646 sizeof(u64)); 647 648 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 649 event->header.size += machine->id_hdr_size; 650 err = perf_tool__process_synth_event(tool, event, 651 machine, process); 652 } 653 654 if (!opts->no_bpf_event) { 655 /* Synthesize PERF_RECORD_BPF_EVENT */ 656 *bpf_event = (struct perf_record_bpf_event) { 657 .header = { 658 .type = PERF_RECORD_BPF_EVENT, 659 .size = sizeof(struct perf_record_bpf_event), 660 }, 661 .type = PERF_BPF_EVENT_PROG_LOAD, 662 .flags = 0, 663 .id = info->id, 664 }; 665 memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE); 666 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 667 event->header.size += machine->id_hdr_size; 668 669 /* save bpf_prog_info to env */ 670 info_node = malloc(sizeof(struct bpf_prog_info_node)); 671 if (!info_node) { 672 err = -1; 673 goto out; 674 } 675 676 info_node->info_linear = info_linear; 677 info_node->metadata = NULL; 678 if (!perf_env__insert_bpf_prog_info(env, info_node)) { 679 /* 680 * Insert failed, likely because of a duplicate event 681 * made by the sideband thread. Ignore synthesizing the 682 * metadata. 683 */ 684 free(info_node); 685 goto out; 686 } 687 /* info_linear is now owned by info_node and shouldn't be freed below. */ 688 info_linear = NULL; 689 690 /* 691 * process after saving bpf_prog_info to env, so that 692 * required information is ready for look up 693 */ 694 err = perf_tool__process_synth_event(tool, event, 695 machine, process); 696 697 /* Synthesize PERF_RECORD_BPF_METADATA */ 698 metadata = bpf_metadata_create(info); 699 if (metadata != NULL) { 700 err = synthesize_perf_record_bpf_metadata(metadata, 701 tool, process, 702 machine); 703 bpf_metadata_free(metadata); 704 } 705 } 706 707 out: 708 free(info_linear); 709 btf__free(btf); 710 return err ? -1 : 0; 711 } 712 713 struct kallsyms_parse { 714 union perf_event *event; 715 perf_event__handler_t process; 716 struct machine *machine; 717 const struct perf_tool *tool; 718 }; 719 720 static int 721 process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data) 722 { 723 struct machine *machine = data->machine; 724 union perf_event *event = data->event; 725 struct perf_record_ksymbol *ksymbol; 726 int len; 727 728 ksymbol = &event->ksymbol; 729 730 *ksymbol = (struct perf_record_ksymbol) { 731 .header = { 732 .type = PERF_RECORD_KSYMBOL, 733 .size = offsetof(struct perf_record_ksymbol, name), 734 }, 735 .addr = addr, 736 .len = page_size, 737 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 738 .flags = 0, 739 }; 740 741 len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name); 742 ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64)); 743 memset((void *) event + event->header.size, 0, machine->id_hdr_size); 744 event->header.size += machine->id_hdr_size; 745 746 return perf_tool__process_synth_event(data->tool, event, machine, 747 data->process); 748 } 749 750 static int 751 kallsyms_process_symbol(void *data, const char *_name, 752 char type __maybe_unused, u64 start) 753 { 754 char disp[KSYM_NAME_LEN]; 755 const char *module; 756 char *name; 757 unsigned long id; 758 int err = 0; 759 760 module = strchr(_name, '\t'); 761 if (!module) 762 return 0; 763 764 /* We are going after [bpf] module ... */ 765 if (strcmp(module + 1, "[bpf]")) 766 return 0; 767 768 name = memdup(_name, (module - _name) + 1); 769 if (!name) 770 return -ENOMEM; 771 772 name[module - _name] = 0; 773 774 /* .. and only for trampolines and dispatchers */ 775 if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) || 776 (sscanf(name, "bpf_dispatcher_%s", disp) == 1)) 777 err = process_bpf_image(name, start, data); 778 779 free(name); 780 return err; 781 } 782 783 int perf_event__synthesize_bpf_events(struct perf_session *session, 784 perf_event__handler_t process, 785 struct machine *machine, 786 struct record_opts *opts) 787 { 788 const char *kallsyms_filename = "/proc/kallsyms"; 789 struct kallsyms_parse arg; 790 union perf_event *event; 791 __u32 id = 0; 792 int err; 793 int fd; 794 795 if (opts->no_bpf_event) 796 return 0; 797 798 event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size); 799 if (!event) 800 return -1; 801 802 /* Synthesize all the bpf programs in system. */ 803 while (true) { 804 err = bpf_prog_get_next_id(id, &id); 805 if (err) { 806 if (errno == ENOENT) { 807 err = 0; 808 break; 809 } 810 /* don't report error on old kernel or EPERM */ 811 err = (errno == EINVAL || errno == EPERM) ? 0 : -1; 812 pr_debug("%s: can\'t get next program: %m%s\n", 813 __func__, errno == EINVAL ? " -- kernel too old?" : ""); 814 break; 815 } 816 fd = bpf_prog_get_fd_by_id(id); 817 if (fd < 0) { 818 pr_debug("%s: failed to get fd for prog_id %u\n", 819 __func__, id); 820 continue; 821 } 822 823 err = perf_event__synthesize_one_bpf_prog(session, process, 824 machine, fd, 825 event, opts); 826 close(fd); 827 if (err) { 828 /* do not return error for old kernel */ 829 if (err == -2) 830 err = 0; 831 break; 832 } 833 } 834 835 /* Synthesize all the bpf images - trampolines/dispatchers. */ 836 if (symbol_conf.kallsyms_name != NULL) 837 kallsyms_filename = symbol_conf.kallsyms_name; 838 839 arg = (struct kallsyms_parse) { 840 .event = event, 841 .process = process, 842 .machine = machine, 843 .tool = session->tool, 844 }; 845 846 if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) 847 pr_err("%s: failed to synthesize bpf images: %m\n", __func__); 848 849 free(event); 850 return err; 851 } 852 853 static int perf_env__add_bpf_info(struct perf_env *env, u32 id) 854 { 855 struct bpf_prog_info_node *info_node; 856 struct perf_bpil *info_linear; 857 struct btf *btf = NULL; 858 u64 arrays; 859 u32 btf_id; 860 int fd, err = 0; 861 862 fd = bpf_prog_get_fd_by_id(id); 863 if (fd < 0) 864 return -EINVAL; 865 866 arrays = 1UL << PERF_BPIL_JITED_KSYMS; 867 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; 868 arrays |= 1UL << PERF_BPIL_FUNC_INFO; 869 arrays |= 1UL << PERF_BPIL_PROG_TAGS; 870 arrays |= 1UL << PERF_BPIL_JITED_INSNS; 871 arrays |= 1UL << PERF_BPIL_LINE_INFO; 872 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; 873 arrays |= 1UL << PERF_BPIL_MAP_IDS; 874 875 info_linear = get_bpf_prog_info_linear(fd, arrays); 876 if (IS_ERR_OR_NULL(info_linear)) { 877 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 878 err = PTR_ERR(info_linear); 879 goto out; 880 } 881 882 btf_id = info_linear->info.btf_id; 883 884 info_node = malloc(sizeof(struct bpf_prog_info_node)); 885 if (info_node) { 886 info_node->info_linear = info_linear; 887 info_node->metadata = bpf_metadata_create(&info_linear->info); 888 if (!perf_env__insert_bpf_prog_info(env, info_node)) { 889 pr_debug("%s: duplicate add bpf info request for id %u\n", 890 __func__, btf_id); 891 bpf_metadata_free(info_node->metadata); 892 free(info_linear); 893 free(info_node); 894 goto out; 895 } 896 } else { 897 free(info_linear); 898 err = -ENOMEM; 899 goto out; 900 } 901 902 if (btf_id == 0) 903 goto out; 904 905 btf = btf__load_from_kernel_by_id(btf_id); 906 if (!btf) { 907 err = -errno; 908 pr_debug("%s: failed to get BTF of id %u %d\n", __func__, btf_id, err); 909 } else { 910 perf_env__fetch_btf(env, btf_id, btf); 911 } 912 913 out: 914 btf__free(btf); 915 close(fd); 916 return err; 917 } 918 919 static int bpf_event__sb_cb(union perf_event *event, void *data) 920 { 921 struct perf_env *env = data; 922 int ret = 0; 923 924 if (event->header.type != PERF_RECORD_BPF_EVENT) 925 return -1; 926 927 switch (event->bpf.type) { 928 case PERF_BPF_EVENT_PROG_LOAD: 929 ret = perf_env__add_bpf_info(env, event->bpf.id); 930 931 case PERF_BPF_EVENT_PROG_UNLOAD: 932 /* 933 * Do not free bpf_prog_info and btf of the program here, 934 * as annotation still need them. They will be freed at 935 * the end of the session. 936 */ 937 break; 938 default: 939 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 940 break; 941 } 942 943 return ret; 944 } 945 946 int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) 947 { 948 struct perf_event_attr attr = { 949 .type = PERF_TYPE_SOFTWARE, 950 .config = PERF_COUNT_SW_DUMMY, 951 .sample_id_all = 1, 952 .watermark = 1, 953 .bpf_event = 1, 954 .size = sizeof(attr), /* to capture ABI version */ 955 }; 956 957 /* 958 * Older gcc versions don't support designated initializers, like above, 959 * for unnamed union members, such as the following: 960 */ 961 attr.wakeup_watermark = 1; 962 963 return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); 964 } 965 966 void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, 967 struct perf_env *env, 968 FILE *fp) 969 { 970 struct bpf_prog_info *info = &info_linear->info; 971 __u64 required_arrays = (1UL << PERF_BPIL_JITED_KSYMS) | 972 (1UL << PERF_BPIL_JITED_FUNC_LENS); 973 __u32 *prog_lens; 974 __u64 *prog_addrs; 975 char name[KSYM_NAME_LEN]; 976 struct btf *btf = NULL; 977 u32 sub_prog_cnt, i; 978 979 sub_prog_cnt = info->nr_jited_ksyms; 980 if (sub_prog_cnt != info->nr_prog_tags || 981 sub_prog_cnt != info->nr_jited_func_lens) 982 return; 983 984 /* Ensure the arrays were present and converted by bpil_offs_to_addr() */ 985 if ((info_linear->arrays & required_arrays) != required_arrays) 986 return; 987 988 prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 989 prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 990 991 if (info->btf_id) { 992 struct btf_node *node; 993 994 node = __perf_env__find_btf(env, info->btf_id); 995 if (node) 996 btf = btf__new((__u8 *)(node->data), 997 node->data_size); 998 } 999 1000 if (sub_prog_cnt == 1) { 1001 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0); 1002 fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n", 1003 info->id, name, prog_addrs[0], prog_lens[0]); 1004 goto out; 1005 } 1006 1007 fprintf(fp, "# bpf_prog_info %u:\n", info->id); 1008 for (i = 0; i < sub_prog_cnt; i++) { 1009 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i); 1010 1011 fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n", 1012 i, name, prog_addrs[i], prog_lens[i]); 1013 } 1014 out: 1015 btf__free(btf); 1016 } 1017