1 // SPDX-License-Identifier: GPL-2.0 2 #include <errno.h> 3 #include <stdlib.h> 4 #include <bpf/bpf.h> 5 #include <bpf/btf.h> 6 #include <bpf/libbpf.h> 7 #include <linux/btf.h> 8 #include <linux/err.h> 9 #include <linux/string.h> 10 #include <internal/lib.h> 11 #include <symbol/kallsyms.h> 12 #include "bpf-event.h" 13 #include "debug.h" 14 #include "dso.h" 15 #include "symbol.h" 16 #include "machine.h" 17 #include "env.h" 18 #include "session.h" 19 #include "map.h" 20 #include "evlist.h" 21 #include "record.h" 22 #include "util/synthetic-events.h" 23 24 #define ptr_to_u64(ptr) ((__u64)(unsigned long)(ptr)) 25 26 static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) 27 { 28 int ret = 0; 29 size_t i; 30 31 for (i = 0; i < len; i++) 32 ret += snprintf(buf + ret, size - ret, "%02x", data[i]); 33 return ret; 34 } 35 36 static int machine__process_bpf_event_load(struct machine *machine, 37 union perf_event *event, 38 struct perf_sample *sample __maybe_unused) 39 { 40 struct bpf_prog_info_linear *info_linear; 41 struct bpf_prog_info_node *info_node; 42 struct perf_env *env = machine->env; 43 int id = event->bpf.id; 44 unsigned int i; 45 46 /* perf-record, no need to handle bpf-event */ 47 if (env == NULL) 48 return 0; 49 50 info_node = perf_env__find_bpf_prog_info(env, id); 51 if (!info_node) 52 return 0; 53 info_linear = info_node->info_linear; 54 55 for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { 56 u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); 57 u64 addr = addrs[i]; 58 struct map *map = maps__find(&machine->kmaps, addr); 59 60 if (map) { 61 map->dso->binary_type = DSO_BINARY_TYPE__BPF_PROG_INFO; 62 map->dso->bpf_prog.id = id; 63 map->dso->bpf_prog.sub_id = i; 64 map->dso->bpf_prog.env = env; 65 } 66 } 67 return 0; 68 } 69 70 int machine__process_bpf(struct machine *machine, union perf_event *event, 71 struct perf_sample *sample) 72 { 73 if (dump_trace) 74 perf_event__fprintf_bpf(event, stdout); 75 76 switch (event->bpf.type) { 77 case PERF_BPF_EVENT_PROG_LOAD: 78 return machine__process_bpf_event_load(machine, event, sample); 79 80 case PERF_BPF_EVENT_PROG_UNLOAD: 81 /* 82 * Do not free bpf_prog_info and btf of the program here, 83 * as annotation still need them. They will be freed at 84 * the end of the session. 85 */ 86 break; 87 default: 88 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 89 break; 90 } 91 return 0; 92 } 93 94 static int perf_env__fetch_btf(struct perf_env *env, 95 u32 btf_id, 96 struct btf *btf) 97 { 98 struct btf_node *node; 99 u32 data_size; 100 const void *data; 101 102 data = btf__get_raw_data(btf, &data_size); 103 104 node = malloc(data_size + sizeof(struct btf_node)); 105 if (!node) 106 return -1; 107 108 node->id = btf_id; 109 node->data_size = data_size; 110 memcpy(node->data, data, data_size); 111 112 perf_env__insert_btf(env, node); 113 return 0; 114 } 115 116 static int synthesize_bpf_prog_name(char *buf, int size, 117 struct bpf_prog_info *info, 118 struct btf *btf, 119 u32 sub_id) 120 { 121 u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags); 122 void *func_infos = (void *)(uintptr_t)(info->func_info); 123 u32 sub_prog_cnt = info->nr_jited_ksyms; 124 const struct bpf_func_info *finfo; 125 const char *short_name = NULL; 126 const struct btf_type *t; 127 int name_len; 128 129 name_len = snprintf(buf, size, "bpf_prog_"); 130 name_len += snprintf_hex(buf + name_len, size - name_len, 131 prog_tags[sub_id], BPF_TAG_SIZE); 132 if (btf) { 133 finfo = func_infos + sub_id * info->func_info_rec_size; 134 t = btf__type_by_id(btf, finfo->type_id); 135 short_name = btf__name_by_offset(btf, t->name_off); 136 } else if (sub_id == 0 && sub_prog_cnt == 1) { 137 /* no subprog */ 138 if (info->name[0]) 139 short_name = info->name; 140 } else 141 short_name = "F"; 142 if (short_name) 143 name_len += snprintf(buf + name_len, size - name_len, 144 "_%s", short_name); 145 return name_len; 146 } 147 148 /* 149 * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf 150 * program. One PERF_RECORD_BPF_EVENT is generated for the program. And 151 * one PERF_RECORD_KSYMBOL is generated for each sub program. 152 * 153 * Returns: 154 * 0 for success; 155 * -1 for failures; 156 * -2 for lack of kernel support. 157 */ 158 static int perf_event__synthesize_one_bpf_prog(struct perf_session *session, 159 perf_event__handler_t process, 160 struct machine *machine, 161 int fd, 162 union perf_event *event, 163 struct record_opts *opts) 164 { 165 struct perf_record_ksymbol *ksymbol_event = &event->ksymbol; 166 struct perf_record_bpf_event *bpf_event = &event->bpf; 167 struct bpf_prog_info_linear *info_linear; 168 struct perf_tool *tool = session->tool; 169 struct bpf_prog_info_node *info_node; 170 struct bpf_prog_info *info; 171 struct btf *btf = NULL; 172 struct perf_env *env; 173 u32 sub_prog_cnt, i; 174 int err = 0; 175 u64 arrays; 176 177 /* 178 * for perf-record and perf-report use header.env; 179 * otherwise, use global perf_env. 180 */ 181 env = session->data ? &session->header.env : &perf_env; 182 183 arrays = 1UL << BPF_PROG_INFO_JITED_KSYMS; 184 arrays |= 1UL << BPF_PROG_INFO_JITED_FUNC_LENS; 185 arrays |= 1UL << BPF_PROG_INFO_FUNC_INFO; 186 arrays |= 1UL << BPF_PROG_INFO_PROG_TAGS; 187 arrays |= 1UL << BPF_PROG_INFO_JITED_INSNS; 188 arrays |= 1UL << BPF_PROG_INFO_LINE_INFO; 189 arrays |= 1UL << BPF_PROG_INFO_JITED_LINE_INFO; 190 191 info_linear = bpf_program__get_prog_info_linear(fd, arrays); 192 if (IS_ERR_OR_NULL(info_linear)) { 193 info_linear = NULL; 194 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 195 return -1; 196 } 197 198 if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) { 199 free(info_linear); 200 pr_debug("%s: the kernel is too old, aborting\n", __func__); 201 return -2; 202 } 203 204 info = &info_linear->info; 205 if (!info->jited_ksyms) { 206 free(info_linear); 207 return -1; 208 } 209 210 /* number of ksyms, func_lengths, and tags should match */ 211 sub_prog_cnt = info->nr_jited_ksyms; 212 if (sub_prog_cnt != info->nr_prog_tags || 213 sub_prog_cnt != info->nr_jited_func_lens) { 214 free(info_linear); 215 return -1; 216 } 217 218 /* check BTF func info support */ 219 if (info->btf_id && info->nr_func_info && info->func_info_rec_size) { 220 /* btf func info number should be same as sub_prog_cnt */ 221 if (sub_prog_cnt != info->nr_func_info) { 222 pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__); 223 free(info_linear); 224 return -1; 225 } 226 if (btf__get_from_id(info->btf_id, &btf)) { 227 pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id); 228 err = -1; 229 btf = NULL; 230 goto out; 231 } 232 perf_env__fetch_btf(env, info->btf_id, btf); 233 } 234 235 /* Synthesize PERF_RECORD_KSYMBOL */ 236 for (i = 0; i < sub_prog_cnt; i++) { 237 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 238 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 239 int name_len; 240 241 *ksymbol_event = (struct perf_record_ksymbol) { 242 .header = { 243 .type = PERF_RECORD_KSYMBOL, 244 .size = offsetof(struct perf_record_ksymbol, name), 245 }, 246 .addr = prog_addrs[i], 247 .len = prog_lens[i], 248 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 249 .flags = 0, 250 }; 251 252 name_len = synthesize_bpf_prog_name(ksymbol_event->name, 253 KSYM_NAME_LEN, info, btf, i); 254 ksymbol_event->header.size += PERF_ALIGN(name_len + 1, 255 sizeof(u64)); 256 257 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 258 event->header.size += machine->id_hdr_size; 259 err = perf_tool__process_synth_event(tool, event, 260 machine, process); 261 } 262 263 if (!opts->no_bpf_event) { 264 /* Synthesize PERF_RECORD_BPF_EVENT */ 265 *bpf_event = (struct perf_record_bpf_event) { 266 .header = { 267 .type = PERF_RECORD_BPF_EVENT, 268 .size = sizeof(struct perf_record_bpf_event), 269 }, 270 .type = PERF_BPF_EVENT_PROG_LOAD, 271 .flags = 0, 272 .id = info->id, 273 }; 274 memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE); 275 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 276 event->header.size += machine->id_hdr_size; 277 278 /* save bpf_prog_info to env */ 279 info_node = malloc(sizeof(struct bpf_prog_info_node)); 280 if (!info_node) { 281 err = -1; 282 goto out; 283 } 284 285 info_node->info_linear = info_linear; 286 perf_env__insert_bpf_prog_info(env, info_node); 287 info_linear = NULL; 288 289 /* 290 * process after saving bpf_prog_info to env, so that 291 * required information is ready for look up 292 */ 293 err = perf_tool__process_synth_event(tool, event, 294 machine, process); 295 } 296 297 out: 298 free(info_linear); 299 free(btf); 300 return err ? -1 : 0; 301 } 302 303 struct kallsyms_parse { 304 union perf_event *event; 305 perf_event__handler_t process; 306 struct machine *machine; 307 struct perf_tool *tool; 308 }; 309 310 static int 311 process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data) 312 { 313 struct machine *machine = data->machine; 314 union perf_event *event = data->event; 315 struct perf_record_ksymbol *ksymbol; 316 int len; 317 318 ksymbol = &event->ksymbol; 319 320 *ksymbol = (struct perf_record_ksymbol) { 321 .header = { 322 .type = PERF_RECORD_KSYMBOL, 323 .size = offsetof(struct perf_record_ksymbol, name), 324 }, 325 .addr = addr, 326 .len = page_size, 327 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 328 .flags = 0, 329 }; 330 331 len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name); 332 ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64)); 333 memset((void *) event + event->header.size, 0, machine->id_hdr_size); 334 event->header.size += machine->id_hdr_size; 335 336 return perf_tool__process_synth_event(data->tool, event, machine, 337 data->process); 338 } 339 340 static int 341 kallsyms_process_symbol(void *data, const char *_name, 342 char type __maybe_unused, u64 start) 343 { 344 char disp[KSYM_NAME_LEN]; 345 char *module, *name; 346 unsigned long id; 347 int err = 0; 348 349 module = strchr(_name, '\t'); 350 if (!module) 351 return 0; 352 353 /* We are going after [bpf] module ... */ 354 if (strcmp(module + 1, "[bpf]")) 355 return 0; 356 357 name = memdup(_name, (module - _name) + 1); 358 if (!name) 359 return -ENOMEM; 360 361 name[module - _name] = 0; 362 363 /* .. and only for trampolines and dispatchers */ 364 if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) || 365 (sscanf(name, "bpf_dispatcher_%s", disp) == 1)) 366 err = process_bpf_image(name, start, data); 367 368 free(name); 369 return err; 370 } 371 372 int perf_event__synthesize_bpf_events(struct perf_session *session, 373 perf_event__handler_t process, 374 struct machine *machine, 375 struct record_opts *opts) 376 { 377 const char *kallsyms_filename = "/proc/kallsyms"; 378 struct kallsyms_parse arg; 379 union perf_event *event; 380 __u32 id = 0; 381 int err; 382 int fd; 383 384 event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size); 385 if (!event) 386 return -1; 387 388 /* Synthesize all the bpf programs in system. */ 389 while (true) { 390 err = bpf_prog_get_next_id(id, &id); 391 if (err) { 392 if (errno == ENOENT) { 393 err = 0; 394 break; 395 } 396 pr_debug("%s: can't get next program: %s%s\n", 397 __func__, strerror(errno), 398 errno == EINVAL ? " -- kernel too old?" : ""); 399 /* don't report error on old kernel or EPERM */ 400 err = (errno == EINVAL || errno == EPERM) ? 0 : -1; 401 break; 402 } 403 fd = bpf_prog_get_fd_by_id(id); 404 if (fd < 0) { 405 pr_debug("%s: failed to get fd for prog_id %u\n", 406 __func__, id); 407 continue; 408 } 409 410 err = perf_event__synthesize_one_bpf_prog(session, process, 411 machine, fd, 412 event, opts); 413 close(fd); 414 if (err) { 415 /* do not return error for old kernel */ 416 if (err == -2) 417 err = 0; 418 break; 419 } 420 } 421 422 /* Synthesize all the bpf images - trampolines/dispatchers. */ 423 if (symbol_conf.kallsyms_name != NULL) 424 kallsyms_filename = symbol_conf.kallsyms_name; 425 426 arg = (struct kallsyms_parse) { 427 .event = event, 428 .process = process, 429 .machine = machine, 430 .tool = session->tool, 431 }; 432 433 if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) { 434 pr_err("%s: failed to synthesize bpf images: %s\n", 435 __func__, strerror(errno)); 436 } 437 438 free(event); 439 return err; 440 } 441 442 static void perf_env__add_bpf_info(struct perf_env *env, u32 id) 443 { 444 struct bpf_prog_info_linear *info_linear; 445 struct bpf_prog_info_node *info_node; 446 struct btf *btf = NULL; 447 u64 arrays; 448 u32 btf_id; 449 int fd; 450 451 fd = bpf_prog_get_fd_by_id(id); 452 if (fd < 0) 453 return; 454 455 arrays = 1UL << BPF_PROG_INFO_JITED_KSYMS; 456 arrays |= 1UL << BPF_PROG_INFO_JITED_FUNC_LENS; 457 arrays |= 1UL << BPF_PROG_INFO_FUNC_INFO; 458 arrays |= 1UL << BPF_PROG_INFO_PROG_TAGS; 459 arrays |= 1UL << BPF_PROG_INFO_JITED_INSNS; 460 arrays |= 1UL << BPF_PROG_INFO_LINE_INFO; 461 arrays |= 1UL << BPF_PROG_INFO_JITED_LINE_INFO; 462 463 info_linear = bpf_program__get_prog_info_linear(fd, arrays); 464 if (IS_ERR_OR_NULL(info_linear)) { 465 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 466 goto out; 467 } 468 469 btf_id = info_linear->info.btf_id; 470 471 info_node = malloc(sizeof(struct bpf_prog_info_node)); 472 if (info_node) { 473 info_node->info_linear = info_linear; 474 perf_env__insert_bpf_prog_info(env, info_node); 475 } else 476 free(info_linear); 477 478 if (btf_id == 0) 479 goto out; 480 481 if (btf__get_from_id(btf_id, &btf)) { 482 pr_debug("%s: failed to get BTF of id %u, aborting\n", 483 __func__, btf_id); 484 goto out; 485 } 486 perf_env__fetch_btf(env, btf_id, btf); 487 488 out: 489 free(btf); 490 close(fd); 491 } 492 493 static int bpf_event__sb_cb(union perf_event *event, void *data) 494 { 495 struct perf_env *env = data; 496 497 if (event->header.type != PERF_RECORD_BPF_EVENT) 498 return -1; 499 500 switch (event->bpf.type) { 501 case PERF_BPF_EVENT_PROG_LOAD: 502 perf_env__add_bpf_info(env, event->bpf.id); 503 504 case PERF_BPF_EVENT_PROG_UNLOAD: 505 /* 506 * Do not free bpf_prog_info and btf of the program here, 507 * as annotation still need them. They will be freed at 508 * the end of the session. 509 */ 510 break; 511 default: 512 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 513 break; 514 } 515 516 return 0; 517 } 518 519 int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) 520 { 521 struct perf_event_attr attr = { 522 .type = PERF_TYPE_SOFTWARE, 523 .config = PERF_COUNT_SW_DUMMY, 524 .sample_id_all = 1, 525 .watermark = 1, 526 .bpf_event = 1, 527 .size = sizeof(attr), /* to capture ABI version */ 528 }; 529 530 /* 531 * Older gcc versions don't support designated initializers, like above, 532 * for unnamed union members, such as the following: 533 */ 534 attr.wakeup_watermark = 1; 535 536 return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); 537 } 538 539 void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, 540 struct perf_env *env, 541 FILE *fp) 542 { 543 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 544 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 545 char name[KSYM_NAME_LEN]; 546 struct btf *btf = NULL; 547 u32 sub_prog_cnt, i; 548 549 sub_prog_cnt = info->nr_jited_ksyms; 550 if (sub_prog_cnt != info->nr_prog_tags || 551 sub_prog_cnt != info->nr_jited_func_lens) 552 return; 553 554 if (info->btf_id) { 555 struct btf_node *node; 556 557 node = perf_env__find_btf(env, info->btf_id); 558 if (node) 559 btf = btf__new((__u8 *)(node->data), 560 node->data_size); 561 } 562 563 if (sub_prog_cnt == 1) { 564 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0); 565 fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n", 566 info->id, name, prog_addrs[0], prog_lens[0]); 567 return; 568 } 569 570 fprintf(fp, "# bpf_prog_info %u:\n", info->id); 571 for (i = 0; i < sub_prog_cnt; i++) { 572 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i); 573 574 fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n", 575 i, name, prog_addrs[i], prog_lens[i]); 576 } 577 } 578