1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ 3 4 #define _GNU_SOURCE 5 #include <errno.h> 6 #include <fcntl.h> 7 #include <signal.h> 8 #include <stdarg.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <time.h> 13 #include <unistd.h> 14 #include <net/if.h> 15 #include <sys/ioctl.h> 16 #include <sys/types.h> 17 #include <sys/stat.h> 18 #include <sys/syscall.h> 19 20 #include <linux/err.h> 21 #include <linux/perf_event.h> 22 #include <linux/sizes.h> 23 24 #include <bpf/bpf.h> 25 #include <bpf/btf.h> 26 #include <bpf/libbpf.h> 27 28 #include "cfg.h" 29 #include "main.h" 30 #include "xlated_dumper.h" 31 32 const char * const prog_type_name[] = { 33 [BPF_PROG_TYPE_UNSPEC] = "unspec", 34 [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", 35 [BPF_PROG_TYPE_KPROBE] = "kprobe", 36 [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls", 37 [BPF_PROG_TYPE_SCHED_ACT] = "sched_act", 38 [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint", 39 [BPF_PROG_TYPE_XDP] = "xdp", 40 [BPF_PROG_TYPE_PERF_EVENT] = "perf_event", 41 [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb", 42 [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock", 43 [BPF_PROG_TYPE_LWT_IN] = "lwt_in", 44 [BPF_PROG_TYPE_LWT_OUT] = "lwt_out", 45 [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", 46 [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", 47 [BPF_PROG_TYPE_SK_SKB] = "sk_skb", 48 [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device", 49 [BPF_PROG_TYPE_SK_MSG] = "sk_msg", 50 [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", 51 [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr", 52 [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local", 53 [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2", 54 [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport", 55 [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector", 56 [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl", 57 [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable", 58 [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt", 59 [BPF_PROG_TYPE_TRACING] = "tracing", 60 [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops", 61 [BPF_PROG_TYPE_EXT] = "ext", 62 }; 63 64 const size_t prog_type_name_size = ARRAY_SIZE(prog_type_name); 65 66 enum dump_mode { 67 DUMP_JITED, 68 DUMP_XLATED, 69 }; 70 71 static const char * const attach_type_strings[] = { 72 [BPF_SK_SKB_STREAM_PARSER] = "stream_parser", 73 [BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict", 74 [BPF_SK_MSG_VERDICT] = "msg_verdict", 75 [BPF_FLOW_DISSECTOR] = "flow_dissector", 76 [__MAX_BPF_ATTACH_TYPE] = NULL, 77 }; 78 79 static enum bpf_attach_type parse_attach_type(const char *str) 80 { 81 enum bpf_attach_type type; 82 83 for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) { 84 if (attach_type_strings[type] && 85 is_prefix(str, attach_type_strings[type])) 86 return type; 87 } 88 89 return __MAX_BPF_ATTACH_TYPE; 90 } 91 92 static void print_boot_time(__u64 nsecs, char *buf, unsigned int size) 93 { 94 struct timespec real_time_ts, boot_time_ts; 95 time_t wallclock_secs; 96 struct tm load_tm; 97 98 buf[--size] = '\0'; 99 100 if (clock_gettime(CLOCK_REALTIME, &real_time_ts) || 101 clock_gettime(CLOCK_BOOTTIME, &boot_time_ts)) { 102 perror("Can't read clocks"); 103 snprintf(buf, size, "%llu", nsecs / 1000000000); 104 return; 105 } 106 107 wallclock_secs = (real_time_ts.tv_sec - boot_time_ts.tv_sec) + 108 (real_time_ts.tv_nsec - boot_time_ts.tv_nsec + nsecs) / 109 1000000000; 110 111 112 if (!localtime_r(&wallclock_secs, &load_tm)) { 113 snprintf(buf, size, "%llu", nsecs / 1000000000); 114 return; 115 } 116 117 if (json_output) 118 strftime(buf, size, "%s", &load_tm); 119 else 120 strftime(buf, size, "%FT%T%z", &load_tm); 121 } 122 123 static void show_prog_maps(int fd, __u32 num_maps) 124 { 125 struct bpf_prog_info info = {}; 126 __u32 len = sizeof(info); 127 __u32 map_ids[num_maps]; 128 unsigned int i; 129 int err; 130 131 info.nr_map_ids = num_maps; 132 info.map_ids = ptr_to_u64(map_ids); 133 134 err = bpf_obj_get_info_by_fd(fd, &info, &len); 135 if (err || !info.nr_map_ids) 136 return; 137 138 if (json_output) { 139 jsonw_name(json_wtr, "map_ids"); 140 jsonw_start_array(json_wtr); 141 for (i = 0; i < info.nr_map_ids; i++) 142 jsonw_uint(json_wtr, map_ids[i]); 143 jsonw_end_array(json_wtr); 144 } else { 145 printf(" map_ids "); 146 for (i = 0; i < info.nr_map_ids; i++) 147 printf("%u%s", map_ids[i], 148 i == info.nr_map_ids - 1 ? "" : ","); 149 } 150 } 151 152 static void print_prog_header_json(struct bpf_prog_info *info) 153 { 154 jsonw_uint_field(json_wtr, "id", info->id); 155 if (info->type < ARRAY_SIZE(prog_type_name)) 156 jsonw_string_field(json_wtr, "type", 157 prog_type_name[info->type]); 158 else 159 jsonw_uint_field(json_wtr, "type", info->type); 160 161 if (*info->name) 162 jsonw_string_field(json_wtr, "name", info->name); 163 164 jsonw_name(json_wtr, "tag"); 165 jsonw_printf(json_wtr, "\"" BPF_TAG_FMT "\"", 166 info->tag[0], info->tag[1], info->tag[2], info->tag[3], 167 info->tag[4], info->tag[5], info->tag[6], info->tag[7]); 168 169 jsonw_bool_field(json_wtr, "gpl_compatible", info->gpl_compatible); 170 if (info->run_time_ns) { 171 jsonw_uint_field(json_wtr, "run_time_ns", info->run_time_ns); 172 jsonw_uint_field(json_wtr, "run_cnt", info->run_cnt); 173 } 174 } 175 176 static void print_prog_json(struct bpf_prog_info *info, int fd) 177 { 178 char *memlock; 179 180 jsonw_start_object(json_wtr); 181 print_prog_header_json(info); 182 print_dev_json(info->ifindex, info->netns_dev, info->netns_ino); 183 184 if (info->load_time) { 185 char buf[32]; 186 187 print_boot_time(info->load_time, buf, sizeof(buf)); 188 189 /* Piggy back on load_time, since 0 uid is a valid one */ 190 jsonw_name(json_wtr, "loaded_at"); 191 jsonw_printf(json_wtr, "%s", buf); 192 jsonw_uint_field(json_wtr, "uid", info->created_by_uid); 193 } 194 195 jsonw_uint_field(json_wtr, "bytes_xlated", info->xlated_prog_len); 196 197 if (info->jited_prog_len) { 198 jsonw_bool_field(json_wtr, "jited", true); 199 jsonw_uint_field(json_wtr, "bytes_jited", info->jited_prog_len); 200 } else { 201 jsonw_bool_field(json_wtr, "jited", false); 202 } 203 204 memlock = get_fdinfo(fd, "memlock"); 205 if (memlock) 206 jsonw_int_field(json_wtr, "bytes_memlock", atoi(memlock)); 207 free(memlock); 208 209 if (info->nr_map_ids) 210 show_prog_maps(fd, info->nr_map_ids); 211 212 if (info->btf_id) 213 jsonw_int_field(json_wtr, "btf_id", info->btf_id); 214 215 if (!hash_empty(prog_table.table)) { 216 struct pinned_obj *obj; 217 218 jsonw_name(json_wtr, "pinned"); 219 jsonw_start_array(json_wtr); 220 hash_for_each_possible(prog_table.table, obj, hash, info->id) { 221 if (obj->id == info->id) 222 jsonw_string(json_wtr, obj->path); 223 } 224 jsonw_end_array(json_wtr); 225 } 226 227 emit_obj_refs_json(&refs_table, info->id, json_wtr); 228 229 jsonw_end_object(json_wtr); 230 } 231 232 static void print_prog_header_plain(struct bpf_prog_info *info) 233 { 234 printf("%u: ", info->id); 235 if (info->type < ARRAY_SIZE(prog_type_name)) 236 printf("%s ", prog_type_name[info->type]); 237 else 238 printf("type %u ", info->type); 239 240 if (*info->name) 241 printf("name %s ", info->name); 242 243 printf("tag "); 244 fprint_hex(stdout, info->tag, BPF_TAG_SIZE, ""); 245 print_dev_plain(info->ifindex, info->netns_dev, info->netns_ino); 246 printf("%s", info->gpl_compatible ? " gpl" : ""); 247 if (info->run_time_ns) 248 printf(" run_time_ns %lld run_cnt %lld", 249 info->run_time_ns, info->run_cnt); 250 printf("\n"); 251 } 252 253 static void print_prog_plain(struct bpf_prog_info *info, int fd) 254 { 255 char *memlock; 256 257 print_prog_header_plain(info); 258 259 if (info->load_time) { 260 char buf[32]; 261 262 print_boot_time(info->load_time, buf, sizeof(buf)); 263 264 /* Piggy back on load_time, since 0 uid is a valid one */ 265 printf("\tloaded_at %s uid %u\n", buf, info->created_by_uid); 266 } 267 268 printf("\txlated %uB", info->xlated_prog_len); 269 270 if (info->jited_prog_len) 271 printf(" jited %uB", info->jited_prog_len); 272 else 273 printf(" not jited"); 274 275 memlock = get_fdinfo(fd, "memlock"); 276 if (memlock) 277 printf(" memlock %sB", memlock); 278 free(memlock); 279 280 if (info->nr_map_ids) 281 show_prog_maps(fd, info->nr_map_ids); 282 283 if (!hash_empty(prog_table.table)) { 284 struct pinned_obj *obj; 285 286 hash_for_each_possible(prog_table.table, obj, hash, info->id) { 287 if (obj->id == info->id) 288 printf("\n\tpinned %s", obj->path); 289 } 290 } 291 292 if (info->btf_id) 293 printf("\n\tbtf_id %d", info->btf_id); 294 295 emit_obj_refs_plain(&refs_table, info->id, "\n\tpids "); 296 297 printf("\n"); 298 } 299 300 static int show_prog(int fd) 301 { 302 struct bpf_prog_info info = {}; 303 __u32 len = sizeof(info); 304 int err; 305 306 err = bpf_obj_get_info_by_fd(fd, &info, &len); 307 if (err) { 308 p_err("can't get prog info: %s", strerror(errno)); 309 return -1; 310 } 311 312 if (json_output) 313 print_prog_json(&info, fd); 314 else 315 print_prog_plain(&info, fd); 316 317 return 0; 318 } 319 320 static int do_show_subset(int argc, char **argv) 321 { 322 int *fds = NULL; 323 int nb_fds, i; 324 int err = -1; 325 326 fds = malloc(sizeof(int)); 327 if (!fds) { 328 p_err("mem alloc failed"); 329 return -1; 330 } 331 nb_fds = prog_parse_fds(&argc, &argv, &fds); 332 if (nb_fds < 1) 333 goto exit_free; 334 335 if (json_output && nb_fds > 1) 336 jsonw_start_array(json_wtr); /* root array */ 337 for (i = 0; i < nb_fds; i++) { 338 err = show_prog(fds[i]); 339 if (err) { 340 for (; i < nb_fds; i++) 341 close(fds[i]); 342 break; 343 } 344 close(fds[i]); 345 } 346 if (json_output && nb_fds > 1) 347 jsonw_end_array(json_wtr); /* root array */ 348 349 exit_free: 350 free(fds); 351 return err; 352 } 353 354 static int do_show(int argc, char **argv) 355 { 356 __u32 id = 0; 357 int err; 358 int fd; 359 360 if (show_pinned) 361 build_pinned_obj_table(&prog_table, BPF_OBJ_PROG); 362 build_obj_refs_table(&refs_table, BPF_OBJ_PROG); 363 364 if (argc == 2) 365 return do_show_subset(argc, argv); 366 367 if (argc) 368 return BAD_ARG(); 369 370 if (json_output) 371 jsonw_start_array(json_wtr); 372 while (true) { 373 err = bpf_prog_get_next_id(id, &id); 374 if (err) { 375 if (errno == ENOENT) { 376 err = 0; 377 break; 378 } 379 p_err("can't get next program: %s%s", strerror(errno), 380 errno == EINVAL ? " -- kernel too old?" : ""); 381 err = -1; 382 break; 383 } 384 385 fd = bpf_prog_get_fd_by_id(id); 386 if (fd < 0) { 387 if (errno == ENOENT) 388 continue; 389 p_err("can't get prog by id (%u): %s", 390 id, strerror(errno)); 391 err = -1; 392 break; 393 } 394 395 err = show_prog(fd); 396 close(fd); 397 if (err) 398 break; 399 } 400 401 if (json_output) 402 jsonw_end_array(json_wtr); 403 404 delete_obj_refs_table(&refs_table); 405 406 return err; 407 } 408 409 static int 410 prog_dump(struct bpf_prog_info *info, enum dump_mode mode, 411 char *filepath, bool opcodes, bool visual, bool linum) 412 { 413 struct bpf_prog_linfo *prog_linfo = NULL; 414 const char *disasm_opt = NULL; 415 struct dump_data dd = {}; 416 void *func_info = NULL; 417 struct btf *btf = NULL; 418 char func_sig[1024]; 419 unsigned char *buf; 420 __u32 member_len; 421 ssize_t n; 422 int fd; 423 424 if (mode == DUMP_JITED) { 425 if (info->jited_prog_len == 0 || !info->jited_prog_insns) { 426 p_info("no instructions returned"); 427 return -1; 428 } 429 buf = (unsigned char *)(info->jited_prog_insns); 430 member_len = info->jited_prog_len; 431 } else { /* DUMP_XLATED */ 432 if (info->xlated_prog_len == 0 || !info->xlated_prog_insns) { 433 p_err("error retrieving insn dump: kernel.kptr_restrict set?"); 434 return -1; 435 } 436 buf = (unsigned char *)info->xlated_prog_insns; 437 member_len = info->xlated_prog_len; 438 } 439 440 if (info->btf_id && btf__get_from_id(info->btf_id, &btf)) { 441 p_err("failed to get btf"); 442 return -1; 443 } 444 445 func_info = (void *)info->func_info; 446 447 if (info->nr_line_info) { 448 prog_linfo = bpf_prog_linfo__new(info); 449 if (!prog_linfo) 450 p_info("error in processing bpf_line_info. continue without it."); 451 } 452 453 if (filepath) { 454 fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600); 455 if (fd < 0) { 456 p_err("can't open file %s: %s", filepath, 457 strerror(errno)); 458 return -1; 459 } 460 461 n = write(fd, buf, member_len); 462 close(fd); 463 if (n != member_len) { 464 p_err("error writing output file: %s", 465 n < 0 ? strerror(errno) : "short write"); 466 return -1; 467 } 468 469 if (json_output) 470 jsonw_null(json_wtr); 471 } else if (mode == DUMP_JITED) { 472 const char *name = NULL; 473 474 if (info->ifindex) { 475 name = ifindex_to_bfd_params(info->ifindex, 476 info->netns_dev, 477 info->netns_ino, 478 &disasm_opt); 479 if (!name) 480 return -1; 481 } 482 483 if (info->nr_jited_func_lens && info->jited_func_lens) { 484 struct kernel_sym *sym = NULL; 485 struct bpf_func_info *record; 486 char sym_name[SYM_MAX_NAME]; 487 unsigned char *img = buf; 488 __u64 *ksyms = NULL; 489 __u32 *lens; 490 __u32 i; 491 if (info->nr_jited_ksyms) { 492 kernel_syms_load(&dd); 493 ksyms = (__u64 *) info->jited_ksyms; 494 } 495 496 if (json_output) 497 jsonw_start_array(json_wtr); 498 499 lens = (__u32 *) info->jited_func_lens; 500 for (i = 0; i < info->nr_jited_func_lens; i++) { 501 if (ksyms) { 502 sym = kernel_syms_search(&dd, ksyms[i]); 503 if (sym) 504 sprintf(sym_name, "%s", sym->name); 505 else 506 sprintf(sym_name, "0x%016llx", ksyms[i]); 507 } else { 508 strcpy(sym_name, "unknown"); 509 } 510 511 if (func_info) { 512 record = func_info + i * info->func_info_rec_size; 513 btf_dumper_type_only(btf, record->type_id, 514 func_sig, 515 sizeof(func_sig)); 516 } 517 518 if (json_output) { 519 jsonw_start_object(json_wtr); 520 if (func_info && func_sig[0] != '\0') { 521 jsonw_name(json_wtr, "proto"); 522 jsonw_string(json_wtr, func_sig); 523 } 524 jsonw_name(json_wtr, "name"); 525 jsonw_string(json_wtr, sym_name); 526 jsonw_name(json_wtr, "insns"); 527 } else { 528 if (func_info && func_sig[0] != '\0') 529 printf("%s:\n", func_sig); 530 printf("%s:\n", sym_name); 531 } 532 533 disasm_print_insn(img, lens[i], opcodes, 534 name, disasm_opt, btf, 535 prog_linfo, ksyms[i], i, 536 linum); 537 538 img += lens[i]; 539 540 if (json_output) 541 jsonw_end_object(json_wtr); 542 else 543 printf("\n"); 544 } 545 546 if (json_output) 547 jsonw_end_array(json_wtr); 548 } else { 549 disasm_print_insn(buf, member_len, opcodes, name, 550 disasm_opt, btf, NULL, 0, 0, false); 551 } 552 } else if (visual) { 553 if (json_output) 554 jsonw_null(json_wtr); 555 else 556 dump_xlated_cfg(buf, member_len); 557 } else { 558 kernel_syms_load(&dd); 559 dd.nr_jited_ksyms = info->nr_jited_ksyms; 560 dd.jited_ksyms = (__u64 *) info->jited_ksyms; 561 dd.btf = btf; 562 dd.func_info = func_info; 563 dd.finfo_rec_size = info->func_info_rec_size; 564 dd.prog_linfo = prog_linfo; 565 566 if (json_output) 567 dump_xlated_json(&dd, buf, member_len, opcodes, 568 linum); 569 else 570 dump_xlated_plain(&dd, buf, member_len, opcodes, 571 linum); 572 kernel_syms_destroy(&dd); 573 } 574 575 return 0; 576 } 577 578 static int do_dump(int argc, char **argv) 579 { 580 struct bpf_prog_info_linear *info_linear; 581 char *filepath = NULL; 582 bool opcodes = false; 583 bool visual = false; 584 enum dump_mode mode; 585 bool linum = false; 586 int *fds = NULL; 587 int nb_fds, i = 0; 588 int err = -1; 589 __u64 arrays; 590 591 if (is_prefix(*argv, "jited")) { 592 if (disasm_init()) 593 return -1; 594 mode = DUMP_JITED; 595 } else if (is_prefix(*argv, "xlated")) { 596 mode = DUMP_XLATED; 597 } else { 598 p_err("expected 'xlated' or 'jited', got: %s", *argv); 599 return -1; 600 } 601 NEXT_ARG(); 602 603 if (argc < 2) 604 usage(); 605 606 fds = malloc(sizeof(int)); 607 if (!fds) { 608 p_err("mem alloc failed"); 609 return -1; 610 } 611 nb_fds = prog_parse_fds(&argc, &argv, &fds); 612 if (nb_fds < 1) 613 goto exit_free; 614 615 if (is_prefix(*argv, "file")) { 616 NEXT_ARG(); 617 if (!argc) { 618 p_err("expected file path"); 619 goto exit_close; 620 } 621 if (nb_fds > 1) { 622 p_err("several programs matched"); 623 goto exit_close; 624 } 625 626 filepath = *argv; 627 NEXT_ARG(); 628 } else if (is_prefix(*argv, "opcodes")) { 629 opcodes = true; 630 NEXT_ARG(); 631 } else if (is_prefix(*argv, "visual")) { 632 if (nb_fds > 1) { 633 p_err("several programs matched"); 634 goto exit_close; 635 } 636 637 visual = true; 638 NEXT_ARG(); 639 } else if (is_prefix(*argv, "linum")) { 640 linum = true; 641 NEXT_ARG(); 642 } 643 644 if (argc) { 645 usage(); 646 goto exit_close; 647 } 648 649 if (mode == DUMP_JITED) 650 arrays = 1UL << BPF_PROG_INFO_JITED_INSNS; 651 else 652 arrays = 1UL << BPF_PROG_INFO_XLATED_INSNS; 653 654 arrays |= 1UL << BPF_PROG_INFO_JITED_KSYMS; 655 arrays |= 1UL << BPF_PROG_INFO_JITED_FUNC_LENS; 656 arrays |= 1UL << BPF_PROG_INFO_FUNC_INFO; 657 arrays |= 1UL << BPF_PROG_INFO_LINE_INFO; 658 arrays |= 1UL << BPF_PROG_INFO_JITED_LINE_INFO; 659 660 if (json_output && nb_fds > 1) 661 jsonw_start_array(json_wtr); /* root array */ 662 for (i = 0; i < nb_fds; i++) { 663 info_linear = bpf_program__get_prog_info_linear(fds[i], arrays); 664 if (IS_ERR_OR_NULL(info_linear)) { 665 p_err("can't get prog info: %s", strerror(errno)); 666 break; 667 } 668 669 if (json_output && nb_fds > 1) { 670 jsonw_start_object(json_wtr); /* prog object */ 671 print_prog_header_json(&info_linear->info); 672 jsonw_name(json_wtr, "insns"); 673 } else if (nb_fds > 1) { 674 print_prog_header_plain(&info_linear->info); 675 } 676 677 err = prog_dump(&info_linear->info, mode, filepath, opcodes, 678 visual, linum); 679 680 if (json_output && nb_fds > 1) 681 jsonw_end_object(json_wtr); /* prog object */ 682 else if (i != nb_fds - 1 && nb_fds > 1) 683 printf("\n"); 684 685 free(info_linear); 686 if (err) 687 break; 688 close(fds[i]); 689 } 690 if (json_output && nb_fds > 1) 691 jsonw_end_array(json_wtr); /* root array */ 692 693 exit_close: 694 for (; i < nb_fds; i++) 695 close(fds[i]); 696 exit_free: 697 free(fds); 698 return err; 699 } 700 701 static int do_pin(int argc, char **argv) 702 { 703 int err; 704 705 err = do_pin_any(argc, argv, prog_parse_fd); 706 if (!err && json_output) 707 jsonw_null(json_wtr); 708 return err; 709 } 710 711 struct map_replace { 712 int idx; 713 int fd; 714 char *name; 715 }; 716 717 static int map_replace_compar(const void *p1, const void *p2) 718 { 719 const struct map_replace *a = p1, *b = p2; 720 721 return a->idx - b->idx; 722 } 723 724 static int parse_attach_detach_args(int argc, char **argv, int *progfd, 725 enum bpf_attach_type *attach_type, 726 int *mapfd) 727 { 728 if (!REQ_ARGS(3)) 729 return -EINVAL; 730 731 *progfd = prog_parse_fd(&argc, &argv); 732 if (*progfd < 0) 733 return *progfd; 734 735 *attach_type = parse_attach_type(*argv); 736 if (*attach_type == __MAX_BPF_ATTACH_TYPE) { 737 p_err("invalid attach/detach type"); 738 return -EINVAL; 739 } 740 741 if (*attach_type == BPF_FLOW_DISSECTOR) { 742 *mapfd = -1; 743 return 0; 744 } 745 746 NEXT_ARG(); 747 if (!REQ_ARGS(2)) 748 return -EINVAL; 749 750 *mapfd = map_parse_fd(&argc, &argv); 751 if (*mapfd < 0) 752 return *mapfd; 753 754 return 0; 755 } 756 757 static int do_attach(int argc, char **argv) 758 { 759 enum bpf_attach_type attach_type; 760 int err, progfd; 761 int mapfd; 762 763 err = parse_attach_detach_args(argc, argv, 764 &progfd, &attach_type, &mapfd); 765 if (err) 766 return err; 767 768 err = bpf_prog_attach(progfd, mapfd, attach_type, 0); 769 if (err) { 770 p_err("failed prog attach to map"); 771 return -EINVAL; 772 } 773 774 if (json_output) 775 jsonw_null(json_wtr); 776 return 0; 777 } 778 779 static int do_detach(int argc, char **argv) 780 { 781 enum bpf_attach_type attach_type; 782 int err, progfd; 783 int mapfd; 784 785 err = parse_attach_detach_args(argc, argv, 786 &progfd, &attach_type, &mapfd); 787 if (err) 788 return err; 789 790 err = bpf_prog_detach2(progfd, mapfd, attach_type); 791 if (err) { 792 p_err("failed prog detach from map"); 793 return -EINVAL; 794 } 795 796 if (json_output) 797 jsonw_null(json_wtr); 798 return 0; 799 } 800 801 static int check_single_stdin(char *file_data_in, char *file_ctx_in) 802 { 803 if (file_data_in && file_ctx_in && 804 !strcmp(file_data_in, "-") && !strcmp(file_ctx_in, "-")) { 805 p_err("cannot use standard input for both data_in and ctx_in"); 806 return -1; 807 } 808 809 return 0; 810 } 811 812 static int get_run_data(const char *fname, void **data_ptr, unsigned int *size) 813 { 814 size_t block_size = 256; 815 size_t buf_size = block_size; 816 size_t nb_read = 0; 817 void *tmp; 818 FILE *f; 819 820 if (!fname) { 821 *data_ptr = NULL; 822 *size = 0; 823 return 0; 824 } 825 826 if (!strcmp(fname, "-")) 827 f = stdin; 828 else 829 f = fopen(fname, "r"); 830 if (!f) { 831 p_err("failed to open %s: %s", fname, strerror(errno)); 832 return -1; 833 } 834 835 *data_ptr = malloc(block_size); 836 if (!*data_ptr) { 837 p_err("failed to allocate memory for data_in/ctx_in: %s", 838 strerror(errno)); 839 goto err_fclose; 840 } 841 842 while ((nb_read += fread(*data_ptr + nb_read, 1, block_size, f))) { 843 if (feof(f)) 844 break; 845 if (ferror(f)) { 846 p_err("failed to read data_in/ctx_in from %s: %s", 847 fname, strerror(errno)); 848 goto err_free; 849 } 850 if (nb_read > buf_size - block_size) { 851 if (buf_size == UINT32_MAX) { 852 p_err("data_in/ctx_in is too long (max: %d)", 853 UINT32_MAX); 854 goto err_free; 855 } 856 /* No space for fread()-ing next chunk; realloc() */ 857 buf_size *= 2; 858 tmp = realloc(*data_ptr, buf_size); 859 if (!tmp) { 860 p_err("failed to reallocate data_in/ctx_in: %s", 861 strerror(errno)); 862 goto err_free; 863 } 864 *data_ptr = tmp; 865 } 866 } 867 if (f != stdin) 868 fclose(f); 869 870 *size = nb_read; 871 return 0; 872 873 err_free: 874 free(*data_ptr); 875 *data_ptr = NULL; 876 err_fclose: 877 if (f != stdin) 878 fclose(f); 879 return -1; 880 } 881 882 static void hex_print(void *data, unsigned int size, FILE *f) 883 { 884 size_t i, j; 885 char c; 886 887 for (i = 0; i < size; i += 16) { 888 /* Row offset */ 889 fprintf(f, "%07zx\t", i); 890 891 /* Hexadecimal values */ 892 for (j = i; j < i + 16 && j < size; j++) 893 fprintf(f, "%02x%s", *(uint8_t *)(data + j), 894 j % 2 ? " " : ""); 895 for (; j < i + 16; j++) 896 fprintf(f, " %s", j % 2 ? " " : ""); 897 898 /* ASCII values (if relevant), '.' otherwise */ 899 fprintf(f, "| "); 900 for (j = i; j < i + 16 && j < size; j++) { 901 c = *(char *)(data + j); 902 if (c < ' ' || c > '~') 903 c = '.'; 904 fprintf(f, "%c%s", c, j == i + 7 ? " " : ""); 905 } 906 907 fprintf(f, "\n"); 908 } 909 } 910 911 static int 912 print_run_output(void *data, unsigned int size, const char *fname, 913 const char *json_key) 914 { 915 size_t nb_written; 916 FILE *f; 917 918 if (!fname) 919 return 0; 920 921 if (!strcmp(fname, "-")) { 922 f = stdout; 923 if (json_output) { 924 jsonw_name(json_wtr, json_key); 925 print_data_json(data, size); 926 } else { 927 hex_print(data, size, f); 928 } 929 return 0; 930 } 931 932 f = fopen(fname, "w"); 933 if (!f) { 934 p_err("failed to open %s: %s", fname, strerror(errno)); 935 return -1; 936 } 937 938 nb_written = fwrite(data, 1, size, f); 939 fclose(f); 940 if (nb_written != size) { 941 p_err("failed to write output data/ctx: %s", strerror(errno)); 942 return -1; 943 } 944 945 return 0; 946 } 947 948 static int alloc_run_data(void **data_ptr, unsigned int size_out) 949 { 950 *data_ptr = calloc(size_out, 1); 951 if (!*data_ptr) { 952 p_err("failed to allocate memory for output data/ctx: %s", 953 strerror(errno)); 954 return -1; 955 } 956 957 return 0; 958 } 959 960 static int do_run(int argc, char **argv) 961 { 962 char *data_fname_in = NULL, *data_fname_out = NULL; 963 char *ctx_fname_in = NULL, *ctx_fname_out = NULL; 964 struct bpf_prog_test_run_attr test_attr = {0}; 965 const unsigned int default_size = SZ_32K; 966 void *data_in = NULL, *data_out = NULL; 967 void *ctx_in = NULL, *ctx_out = NULL; 968 unsigned int repeat = 1; 969 int fd, err; 970 971 if (!REQ_ARGS(4)) 972 return -1; 973 974 fd = prog_parse_fd(&argc, &argv); 975 if (fd < 0) 976 return -1; 977 978 while (argc) { 979 if (detect_common_prefix(*argv, "data_in", "data_out", 980 "data_size_out", NULL)) 981 return -1; 982 if (detect_common_prefix(*argv, "ctx_in", "ctx_out", 983 "ctx_size_out", NULL)) 984 return -1; 985 986 if (is_prefix(*argv, "data_in")) { 987 NEXT_ARG(); 988 if (!REQ_ARGS(1)) 989 return -1; 990 991 data_fname_in = GET_ARG(); 992 if (check_single_stdin(data_fname_in, ctx_fname_in)) 993 return -1; 994 } else if (is_prefix(*argv, "data_out")) { 995 NEXT_ARG(); 996 if (!REQ_ARGS(1)) 997 return -1; 998 999 data_fname_out = GET_ARG(); 1000 } else if (is_prefix(*argv, "data_size_out")) { 1001 char *endptr; 1002 1003 NEXT_ARG(); 1004 if (!REQ_ARGS(1)) 1005 return -1; 1006 1007 test_attr.data_size_out = strtoul(*argv, &endptr, 0); 1008 if (*endptr) { 1009 p_err("can't parse %s as output data size", 1010 *argv); 1011 return -1; 1012 } 1013 NEXT_ARG(); 1014 } else if (is_prefix(*argv, "ctx_in")) { 1015 NEXT_ARG(); 1016 if (!REQ_ARGS(1)) 1017 return -1; 1018 1019 ctx_fname_in = GET_ARG(); 1020 if (check_single_stdin(data_fname_in, ctx_fname_in)) 1021 return -1; 1022 } else if (is_prefix(*argv, "ctx_out")) { 1023 NEXT_ARG(); 1024 if (!REQ_ARGS(1)) 1025 return -1; 1026 1027 ctx_fname_out = GET_ARG(); 1028 } else if (is_prefix(*argv, "ctx_size_out")) { 1029 char *endptr; 1030 1031 NEXT_ARG(); 1032 if (!REQ_ARGS(1)) 1033 return -1; 1034 1035 test_attr.ctx_size_out = strtoul(*argv, &endptr, 0); 1036 if (*endptr) { 1037 p_err("can't parse %s as output context size", 1038 *argv); 1039 return -1; 1040 } 1041 NEXT_ARG(); 1042 } else if (is_prefix(*argv, "repeat")) { 1043 char *endptr; 1044 1045 NEXT_ARG(); 1046 if (!REQ_ARGS(1)) 1047 return -1; 1048 1049 repeat = strtoul(*argv, &endptr, 0); 1050 if (*endptr) { 1051 p_err("can't parse %s as repeat number", 1052 *argv); 1053 return -1; 1054 } 1055 NEXT_ARG(); 1056 } else { 1057 p_err("expected no more arguments, 'data_in', 'data_out', 'data_size_out', 'ctx_in', 'ctx_out', 'ctx_size_out' or 'repeat', got: '%s'?", 1058 *argv); 1059 return -1; 1060 } 1061 } 1062 1063 err = get_run_data(data_fname_in, &data_in, &test_attr.data_size_in); 1064 if (err) 1065 return -1; 1066 1067 if (data_in) { 1068 if (!test_attr.data_size_out) 1069 test_attr.data_size_out = default_size; 1070 err = alloc_run_data(&data_out, test_attr.data_size_out); 1071 if (err) 1072 goto free_data_in; 1073 } 1074 1075 err = get_run_data(ctx_fname_in, &ctx_in, &test_attr.ctx_size_in); 1076 if (err) 1077 goto free_data_out; 1078 1079 if (ctx_in) { 1080 if (!test_attr.ctx_size_out) 1081 test_attr.ctx_size_out = default_size; 1082 err = alloc_run_data(&ctx_out, test_attr.ctx_size_out); 1083 if (err) 1084 goto free_ctx_in; 1085 } 1086 1087 test_attr.prog_fd = fd; 1088 test_attr.repeat = repeat; 1089 test_attr.data_in = data_in; 1090 test_attr.data_out = data_out; 1091 test_attr.ctx_in = ctx_in; 1092 test_attr.ctx_out = ctx_out; 1093 1094 err = bpf_prog_test_run_xattr(&test_attr); 1095 if (err) { 1096 p_err("failed to run program: %s", strerror(errno)); 1097 goto free_ctx_out; 1098 } 1099 1100 err = 0; 1101 1102 if (json_output) 1103 jsonw_start_object(json_wtr); /* root */ 1104 1105 /* Do not exit on errors occurring when printing output data/context, 1106 * we still want to print return value and duration for program run. 1107 */ 1108 if (test_attr.data_size_out) 1109 err += print_run_output(test_attr.data_out, 1110 test_attr.data_size_out, 1111 data_fname_out, "data_out"); 1112 if (test_attr.ctx_size_out) 1113 err += print_run_output(test_attr.ctx_out, 1114 test_attr.ctx_size_out, 1115 ctx_fname_out, "ctx_out"); 1116 1117 if (json_output) { 1118 jsonw_uint_field(json_wtr, "retval", test_attr.retval); 1119 jsonw_uint_field(json_wtr, "duration", test_attr.duration); 1120 jsonw_end_object(json_wtr); /* root */ 1121 } else { 1122 fprintf(stdout, "Return value: %u, duration%s: %uns\n", 1123 test_attr.retval, 1124 repeat > 1 ? " (average)" : "", test_attr.duration); 1125 } 1126 1127 free_ctx_out: 1128 free(ctx_out); 1129 free_ctx_in: 1130 free(ctx_in); 1131 free_data_out: 1132 free(data_out); 1133 free_data_in: 1134 free(data_in); 1135 1136 return err; 1137 } 1138 1139 static int 1140 get_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 1141 enum bpf_attach_type *expected_attach_type) 1142 { 1143 libbpf_print_fn_t print_backup; 1144 int ret; 1145 1146 ret = libbpf_prog_type_by_name(name, prog_type, expected_attach_type); 1147 if (!ret) 1148 return ret; 1149 1150 /* libbpf_prog_type_by_name() failed, let's re-run with debug level */ 1151 print_backup = libbpf_set_print(print_all_levels); 1152 ret = libbpf_prog_type_by_name(name, prog_type, expected_attach_type); 1153 libbpf_set_print(print_backup); 1154 1155 return ret; 1156 } 1157 1158 static int load_with_options(int argc, char **argv, bool first_prog_only) 1159 { 1160 enum bpf_prog_type common_prog_type = BPF_PROG_TYPE_UNSPEC; 1161 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts, 1162 .relaxed_maps = relaxed_maps, 1163 ); 1164 struct bpf_object_load_attr load_attr = { 0 }; 1165 enum bpf_attach_type expected_attach_type; 1166 struct map_replace *map_replace = NULL; 1167 struct bpf_program *prog = NULL, *pos; 1168 unsigned int old_map_fds = 0; 1169 const char *pinmaps = NULL; 1170 struct bpf_object *obj; 1171 struct bpf_map *map; 1172 const char *pinfile; 1173 unsigned int i, j; 1174 __u32 ifindex = 0; 1175 const char *file; 1176 int idx, err; 1177 1178 1179 if (!REQ_ARGS(2)) 1180 return -1; 1181 file = GET_ARG(); 1182 pinfile = GET_ARG(); 1183 1184 while (argc) { 1185 if (is_prefix(*argv, "type")) { 1186 char *type; 1187 1188 NEXT_ARG(); 1189 1190 if (common_prog_type != BPF_PROG_TYPE_UNSPEC) { 1191 p_err("program type already specified"); 1192 goto err_free_reuse_maps; 1193 } 1194 if (!REQ_ARGS(1)) 1195 goto err_free_reuse_maps; 1196 1197 /* Put a '/' at the end of type to appease libbpf */ 1198 type = malloc(strlen(*argv) + 2); 1199 if (!type) { 1200 p_err("mem alloc failed"); 1201 goto err_free_reuse_maps; 1202 } 1203 *type = 0; 1204 strcat(type, *argv); 1205 strcat(type, "/"); 1206 1207 err = get_prog_type_by_name(type, &common_prog_type, 1208 &expected_attach_type); 1209 free(type); 1210 if (err < 0) 1211 goto err_free_reuse_maps; 1212 1213 NEXT_ARG(); 1214 } else if (is_prefix(*argv, "map")) { 1215 void *new_map_replace; 1216 char *endptr, *name; 1217 int fd; 1218 1219 NEXT_ARG(); 1220 1221 if (!REQ_ARGS(4)) 1222 goto err_free_reuse_maps; 1223 1224 if (is_prefix(*argv, "idx")) { 1225 NEXT_ARG(); 1226 1227 idx = strtoul(*argv, &endptr, 0); 1228 if (*endptr) { 1229 p_err("can't parse %s as IDX", *argv); 1230 goto err_free_reuse_maps; 1231 } 1232 name = NULL; 1233 } else if (is_prefix(*argv, "name")) { 1234 NEXT_ARG(); 1235 1236 name = *argv; 1237 idx = -1; 1238 } else { 1239 p_err("expected 'idx' or 'name', got: '%s'?", 1240 *argv); 1241 goto err_free_reuse_maps; 1242 } 1243 NEXT_ARG(); 1244 1245 fd = map_parse_fd(&argc, &argv); 1246 if (fd < 0) 1247 goto err_free_reuse_maps; 1248 1249 new_map_replace = reallocarray(map_replace, 1250 old_map_fds + 1, 1251 sizeof(*map_replace)); 1252 if (!new_map_replace) { 1253 p_err("mem alloc failed"); 1254 goto err_free_reuse_maps; 1255 } 1256 map_replace = new_map_replace; 1257 1258 map_replace[old_map_fds].idx = idx; 1259 map_replace[old_map_fds].name = name; 1260 map_replace[old_map_fds].fd = fd; 1261 old_map_fds++; 1262 } else if (is_prefix(*argv, "dev")) { 1263 NEXT_ARG(); 1264 1265 if (ifindex) { 1266 p_err("offload device already specified"); 1267 goto err_free_reuse_maps; 1268 } 1269 if (!REQ_ARGS(1)) 1270 goto err_free_reuse_maps; 1271 1272 ifindex = if_nametoindex(*argv); 1273 if (!ifindex) { 1274 p_err("unrecognized netdevice '%s': %s", 1275 *argv, strerror(errno)); 1276 goto err_free_reuse_maps; 1277 } 1278 NEXT_ARG(); 1279 } else if (is_prefix(*argv, "pinmaps")) { 1280 NEXT_ARG(); 1281 1282 if (!REQ_ARGS(1)) 1283 goto err_free_reuse_maps; 1284 1285 pinmaps = GET_ARG(); 1286 } else { 1287 p_err("expected no more arguments, 'type', 'map' or 'dev', got: '%s'?", 1288 *argv); 1289 goto err_free_reuse_maps; 1290 } 1291 } 1292 1293 set_max_rlimit(); 1294 1295 obj = bpf_object__open_file(file, &open_opts); 1296 if (IS_ERR_OR_NULL(obj)) { 1297 p_err("failed to open object file"); 1298 goto err_free_reuse_maps; 1299 } 1300 1301 bpf_object__for_each_program(pos, obj) { 1302 enum bpf_prog_type prog_type = common_prog_type; 1303 1304 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 1305 const char *sec_name = bpf_program__title(pos, false); 1306 1307 err = get_prog_type_by_name(sec_name, &prog_type, 1308 &expected_attach_type); 1309 if (err < 0) 1310 goto err_close_obj; 1311 } 1312 1313 bpf_program__set_ifindex(pos, ifindex); 1314 bpf_program__set_type(pos, prog_type); 1315 bpf_program__set_expected_attach_type(pos, expected_attach_type); 1316 } 1317 1318 qsort(map_replace, old_map_fds, sizeof(*map_replace), 1319 map_replace_compar); 1320 1321 /* After the sort maps by name will be first on the list, because they 1322 * have idx == -1. Resolve them. 1323 */ 1324 j = 0; 1325 while (j < old_map_fds && map_replace[j].name) { 1326 i = 0; 1327 bpf_object__for_each_map(map, obj) { 1328 if (!strcmp(bpf_map__name(map), map_replace[j].name)) { 1329 map_replace[j].idx = i; 1330 break; 1331 } 1332 i++; 1333 } 1334 if (map_replace[j].idx == -1) { 1335 p_err("unable to find map '%s'", map_replace[j].name); 1336 goto err_close_obj; 1337 } 1338 j++; 1339 } 1340 /* Resort if any names were resolved */ 1341 if (j) 1342 qsort(map_replace, old_map_fds, sizeof(*map_replace), 1343 map_replace_compar); 1344 1345 /* Set ifindex and name reuse */ 1346 j = 0; 1347 idx = 0; 1348 bpf_object__for_each_map(map, obj) { 1349 if (!bpf_map__is_offload_neutral(map)) 1350 bpf_map__set_ifindex(map, ifindex); 1351 1352 if (j < old_map_fds && idx == map_replace[j].idx) { 1353 err = bpf_map__reuse_fd(map, map_replace[j++].fd); 1354 if (err) { 1355 p_err("unable to set up map reuse: %d", err); 1356 goto err_close_obj; 1357 } 1358 1359 /* Next reuse wants to apply to the same map */ 1360 if (j < old_map_fds && map_replace[j].idx == idx) { 1361 p_err("replacement for map idx %d specified more than once", 1362 idx); 1363 goto err_close_obj; 1364 } 1365 } 1366 1367 idx++; 1368 } 1369 if (j < old_map_fds) { 1370 p_err("map idx '%d' not used", map_replace[j].idx); 1371 goto err_close_obj; 1372 } 1373 1374 load_attr.obj = obj; 1375 if (verifier_logs) 1376 /* log_level1 + log_level2 + stats, but not stable UAPI */ 1377 load_attr.log_level = 1 + 2 + 4; 1378 1379 err = bpf_object__load_xattr(&load_attr); 1380 if (err) { 1381 p_err("failed to load object file"); 1382 goto err_close_obj; 1383 } 1384 1385 err = mount_bpffs_for_pin(pinfile); 1386 if (err) 1387 goto err_close_obj; 1388 1389 if (first_prog_only) { 1390 prog = bpf_program__next(NULL, obj); 1391 if (!prog) { 1392 p_err("object file doesn't contain any bpf program"); 1393 goto err_close_obj; 1394 } 1395 1396 err = bpf_obj_pin(bpf_program__fd(prog), pinfile); 1397 if (err) { 1398 p_err("failed to pin program %s", 1399 bpf_program__title(prog, false)); 1400 goto err_close_obj; 1401 } 1402 } else { 1403 err = bpf_object__pin_programs(obj, pinfile); 1404 if (err) { 1405 p_err("failed to pin all programs"); 1406 goto err_close_obj; 1407 } 1408 } 1409 1410 if (pinmaps) { 1411 err = bpf_object__pin_maps(obj, pinmaps); 1412 if (err) { 1413 p_err("failed to pin all maps"); 1414 goto err_unpin; 1415 } 1416 } 1417 1418 if (json_output) 1419 jsonw_null(json_wtr); 1420 1421 bpf_object__close(obj); 1422 for (i = 0; i < old_map_fds; i++) 1423 close(map_replace[i].fd); 1424 free(map_replace); 1425 1426 return 0; 1427 1428 err_unpin: 1429 if (first_prog_only) 1430 unlink(pinfile); 1431 else 1432 bpf_object__unpin_programs(obj, pinfile); 1433 err_close_obj: 1434 bpf_object__close(obj); 1435 err_free_reuse_maps: 1436 for (i = 0; i < old_map_fds; i++) 1437 close(map_replace[i].fd); 1438 free(map_replace); 1439 return -1; 1440 } 1441 1442 static int do_load(int argc, char **argv) 1443 { 1444 return load_with_options(argc, argv, true); 1445 } 1446 1447 static int do_loadall(int argc, char **argv) 1448 { 1449 return load_with_options(argc, argv, false); 1450 } 1451 1452 #ifdef BPFTOOL_WITHOUT_SKELETONS 1453 1454 static int do_profile(int argc, char **argv) 1455 { 1456 p_err("bpftool prog profile command is not supported. Please build bpftool with clang >= 10.0.0"); 1457 return 0; 1458 } 1459 1460 #else /* BPFTOOL_WITHOUT_SKELETONS */ 1461 1462 #include "profiler.skel.h" 1463 1464 struct profile_metric { 1465 const char *name; 1466 struct bpf_perf_event_value val; 1467 struct perf_event_attr attr; 1468 bool selected; 1469 1470 /* calculate ratios like instructions per cycle */ 1471 const int ratio_metric; /* 0 for N/A, 1 for index 0 (cycles) */ 1472 const char *ratio_desc; 1473 const float ratio_mul; 1474 } metrics[] = { 1475 { 1476 .name = "cycles", 1477 .attr = { 1478 .type = PERF_TYPE_HARDWARE, 1479 .config = PERF_COUNT_HW_CPU_CYCLES, 1480 .exclude_user = 1, 1481 }, 1482 }, 1483 { 1484 .name = "instructions", 1485 .attr = { 1486 .type = PERF_TYPE_HARDWARE, 1487 .config = PERF_COUNT_HW_INSTRUCTIONS, 1488 .exclude_user = 1, 1489 }, 1490 .ratio_metric = 1, 1491 .ratio_desc = "insns per cycle", 1492 .ratio_mul = 1.0, 1493 }, 1494 { 1495 .name = "l1d_loads", 1496 .attr = { 1497 .type = PERF_TYPE_HW_CACHE, 1498 .config = 1499 PERF_COUNT_HW_CACHE_L1D | 1500 (PERF_COUNT_HW_CACHE_OP_READ << 8) | 1501 (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16), 1502 .exclude_user = 1, 1503 }, 1504 }, 1505 { 1506 .name = "llc_misses", 1507 .attr = { 1508 .type = PERF_TYPE_HW_CACHE, 1509 .config = 1510 PERF_COUNT_HW_CACHE_LL | 1511 (PERF_COUNT_HW_CACHE_OP_READ << 8) | 1512 (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), 1513 .exclude_user = 1 1514 }, 1515 .ratio_metric = 2, 1516 .ratio_desc = "LLC misses per million insns", 1517 .ratio_mul = 1e6, 1518 }, 1519 }; 1520 1521 static __u64 profile_total_count; 1522 1523 #define MAX_NUM_PROFILE_METRICS 4 1524 1525 static int profile_parse_metrics(int argc, char **argv) 1526 { 1527 unsigned int metric_cnt; 1528 int selected_cnt = 0; 1529 unsigned int i; 1530 1531 metric_cnt = sizeof(metrics) / sizeof(struct profile_metric); 1532 1533 while (argc > 0) { 1534 for (i = 0; i < metric_cnt; i++) { 1535 if (is_prefix(argv[0], metrics[i].name)) { 1536 if (!metrics[i].selected) 1537 selected_cnt++; 1538 metrics[i].selected = true; 1539 break; 1540 } 1541 } 1542 if (i == metric_cnt) { 1543 p_err("unknown metric %s", argv[0]); 1544 return -1; 1545 } 1546 NEXT_ARG(); 1547 } 1548 if (selected_cnt > MAX_NUM_PROFILE_METRICS) { 1549 p_err("too many (%d) metrics, please specify no more than %d metrics at at time", 1550 selected_cnt, MAX_NUM_PROFILE_METRICS); 1551 return -1; 1552 } 1553 return selected_cnt; 1554 } 1555 1556 static void profile_read_values(struct profiler_bpf *obj) 1557 { 1558 __u32 m, cpu, num_cpu = obj->rodata->num_cpu; 1559 int reading_map_fd, count_map_fd; 1560 __u64 counts[num_cpu]; 1561 __u32 key = 0; 1562 int err; 1563 1564 reading_map_fd = bpf_map__fd(obj->maps.accum_readings); 1565 count_map_fd = bpf_map__fd(obj->maps.counts); 1566 if (reading_map_fd < 0 || count_map_fd < 0) { 1567 p_err("failed to get fd for map"); 1568 return; 1569 } 1570 1571 err = bpf_map_lookup_elem(count_map_fd, &key, counts); 1572 if (err) { 1573 p_err("failed to read count_map: %s", strerror(errno)); 1574 return; 1575 } 1576 1577 profile_total_count = 0; 1578 for (cpu = 0; cpu < num_cpu; cpu++) 1579 profile_total_count += counts[cpu]; 1580 1581 for (m = 0; m < ARRAY_SIZE(metrics); m++) { 1582 struct bpf_perf_event_value values[num_cpu]; 1583 1584 if (!metrics[m].selected) 1585 continue; 1586 1587 err = bpf_map_lookup_elem(reading_map_fd, &key, values); 1588 if (err) { 1589 p_err("failed to read reading_map: %s", 1590 strerror(errno)); 1591 return; 1592 } 1593 for (cpu = 0; cpu < num_cpu; cpu++) { 1594 metrics[m].val.counter += values[cpu].counter; 1595 metrics[m].val.enabled += values[cpu].enabled; 1596 metrics[m].val.running += values[cpu].running; 1597 } 1598 key++; 1599 } 1600 } 1601 1602 static void profile_print_readings_json(void) 1603 { 1604 __u32 m; 1605 1606 jsonw_start_array(json_wtr); 1607 for (m = 0; m < ARRAY_SIZE(metrics); m++) { 1608 if (!metrics[m].selected) 1609 continue; 1610 jsonw_start_object(json_wtr); 1611 jsonw_string_field(json_wtr, "metric", metrics[m].name); 1612 jsonw_lluint_field(json_wtr, "run_cnt", profile_total_count); 1613 jsonw_lluint_field(json_wtr, "value", metrics[m].val.counter); 1614 jsonw_lluint_field(json_wtr, "enabled", metrics[m].val.enabled); 1615 jsonw_lluint_field(json_wtr, "running", metrics[m].val.running); 1616 1617 jsonw_end_object(json_wtr); 1618 } 1619 jsonw_end_array(json_wtr); 1620 } 1621 1622 static void profile_print_readings_plain(void) 1623 { 1624 __u32 m; 1625 1626 printf("\n%18llu %-20s\n", profile_total_count, "run_cnt"); 1627 for (m = 0; m < ARRAY_SIZE(metrics); m++) { 1628 struct bpf_perf_event_value *val = &metrics[m].val; 1629 int r; 1630 1631 if (!metrics[m].selected) 1632 continue; 1633 printf("%18llu %-20s", val->counter, metrics[m].name); 1634 1635 r = metrics[m].ratio_metric - 1; 1636 if (r >= 0 && metrics[r].selected && 1637 metrics[r].val.counter > 0) { 1638 printf("# %8.2f %-30s", 1639 val->counter * metrics[m].ratio_mul / 1640 metrics[r].val.counter, 1641 metrics[m].ratio_desc); 1642 } else { 1643 printf("%-41s", ""); 1644 } 1645 1646 if (val->enabled > val->running) 1647 printf("(%4.2f%%)", 1648 val->running * 100.0 / val->enabled); 1649 printf("\n"); 1650 } 1651 } 1652 1653 static void profile_print_readings(void) 1654 { 1655 if (json_output) 1656 profile_print_readings_json(); 1657 else 1658 profile_print_readings_plain(); 1659 } 1660 1661 static char *profile_target_name(int tgt_fd) 1662 { 1663 struct bpf_prog_info_linear *info_linear; 1664 struct bpf_func_info *func_info; 1665 const struct btf_type *t; 1666 char *name = NULL; 1667 struct btf *btf; 1668 1669 info_linear = bpf_program__get_prog_info_linear( 1670 tgt_fd, 1UL << BPF_PROG_INFO_FUNC_INFO); 1671 if (IS_ERR_OR_NULL(info_linear)) { 1672 p_err("failed to get info_linear for prog FD %d", tgt_fd); 1673 return NULL; 1674 } 1675 1676 if (info_linear->info.btf_id == 0 || 1677 btf__get_from_id(info_linear->info.btf_id, &btf)) { 1678 p_err("prog FD %d doesn't have valid btf", tgt_fd); 1679 goto out; 1680 } 1681 1682 func_info = (struct bpf_func_info *)(info_linear->info.func_info); 1683 t = btf__type_by_id(btf, func_info[0].type_id); 1684 if (!t) { 1685 p_err("btf %d doesn't have type %d", 1686 info_linear->info.btf_id, func_info[0].type_id); 1687 goto out; 1688 } 1689 name = strdup(btf__name_by_offset(btf, t->name_off)); 1690 out: 1691 free(info_linear); 1692 return name; 1693 } 1694 1695 static struct profiler_bpf *profile_obj; 1696 static int profile_tgt_fd = -1; 1697 static char *profile_tgt_name; 1698 static int *profile_perf_events; 1699 static int profile_perf_event_cnt; 1700 1701 static void profile_close_perf_events(struct profiler_bpf *obj) 1702 { 1703 int i; 1704 1705 for (i = profile_perf_event_cnt - 1; i >= 0; i--) 1706 close(profile_perf_events[i]); 1707 1708 free(profile_perf_events); 1709 profile_perf_event_cnt = 0; 1710 } 1711 1712 static int profile_open_perf_events(struct profiler_bpf *obj) 1713 { 1714 unsigned int cpu, m; 1715 int map_fd, pmu_fd; 1716 1717 profile_perf_events = calloc( 1718 sizeof(int), obj->rodata->num_cpu * obj->rodata->num_metric); 1719 if (!profile_perf_events) { 1720 p_err("failed to allocate memory for perf_event array: %s", 1721 strerror(errno)); 1722 return -1; 1723 } 1724 map_fd = bpf_map__fd(obj->maps.events); 1725 if (map_fd < 0) { 1726 p_err("failed to get fd for events map"); 1727 return -1; 1728 } 1729 1730 for (m = 0; m < ARRAY_SIZE(metrics); m++) { 1731 if (!metrics[m].selected) 1732 continue; 1733 for (cpu = 0; cpu < obj->rodata->num_cpu; cpu++) { 1734 pmu_fd = syscall(__NR_perf_event_open, &metrics[m].attr, 1735 -1/*pid*/, cpu, -1/*group_fd*/, 0); 1736 if (pmu_fd < 0 || 1737 bpf_map_update_elem(map_fd, &profile_perf_event_cnt, 1738 &pmu_fd, BPF_ANY) || 1739 ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0)) { 1740 p_err("failed to create event %s on cpu %d", 1741 metrics[m].name, cpu); 1742 return -1; 1743 } 1744 profile_perf_events[profile_perf_event_cnt++] = pmu_fd; 1745 } 1746 } 1747 return 0; 1748 } 1749 1750 static void profile_print_and_cleanup(void) 1751 { 1752 profile_close_perf_events(profile_obj); 1753 profile_read_values(profile_obj); 1754 profile_print_readings(); 1755 profiler_bpf__destroy(profile_obj); 1756 1757 close(profile_tgt_fd); 1758 free(profile_tgt_name); 1759 } 1760 1761 static void int_exit(int signo) 1762 { 1763 profile_print_and_cleanup(); 1764 exit(0); 1765 } 1766 1767 static int do_profile(int argc, char **argv) 1768 { 1769 int num_metric, num_cpu, err = -1; 1770 struct bpf_program *prog; 1771 unsigned long duration; 1772 char *endptr; 1773 1774 /* we at least need two args for the prog and one metric */ 1775 if (!REQ_ARGS(3)) 1776 return -EINVAL; 1777 1778 /* parse target fd */ 1779 profile_tgt_fd = prog_parse_fd(&argc, &argv); 1780 if (profile_tgt_fd < 0) { 1781 p_err("failed to parse fd"); 1782 return -1; 1783 } 1784 1785 /* parse profiling optional duration */ 1786 if (argc > 2 && is_prefix(argv[0], "duration")) { 1787 NEXT_ARG(); 1788 duration = strtoul(*argv, &endptr, 0); 1789 if (*endptr) 1790 usage(); 1791 NEXT_ARG(); 1792 } else { 1793 duration = UINT_MAX; 1794 } 1795 1796 num_metric = profile_parse_metrics(argc, argv); 1797 if (num_metric <= 0) 1798 goto out; 1799 1800 num_cpu = libbpf_num_possible_cpus(); 1801 if (num_cpu <= 0) { 1802 p_err("failed to identify number of CPUs"); 1803 goto out; 1804 } 1805 1806 profile_obj = profiler_bpf__open(); 1807 if (!profile_obj) { 1808 p_err("failed to open and/or load BPF object"); 1809 goto out; 1810 } 1811 1812 profile_obj->rodata->num_cpu = num_cpu; 1813 profile_obj->rodata->num_metric = num_metric; 1814 1815 /* adjust map sizes */ 1816 bpf_map__resize(profile_obj->maps.events, num_metric * num_cpu); 1817 bpf_map__resize(profile_obj->maps.fentry_readings, num_metric); 1818 bpf_map__resize(profile_obj->maps.accum_readings, num_metric); 1819 bpf_map__resize(profile_obj->maps.counts, 1); 1820 1821 /* change target name */ 1822 profile_tgt_name = profile_target_name(profile_tgt_fd); 1823 if (!profile_tgt_name) 1824 goto out; 1825 1826 bpf_object__for_each_program(prog, profile_obj->obj) { 1827 err = bpf_program__set_attach_target(prog, profile_tgt_fd, 1828 profile_tgt_name); 1829 if (err) { 1830 p_err("failed to set attach target\n"); 1831 goto out; 1832 } 1833 } 1834 1835 set_max_rlimit(); 1836 err = profiler_bpf__load(profile_obj); 1837 if (err) { 1838 p_err("failed to load profile_obj"); 1839 goto out; 1840 } 1841 1842 err = profile_open_perf_events(profile_obj); 1843 if (err) 1844 goto out; 1845 1846 err = profiler_bpf__attach(profile_obj); 1847 if (err) { 1848 p_err("failed to attach profile_obj"); 1849 goto out; 1850 } 1851 signal(SIGINT, int_exit); 1852 1853 sleep(duration); 1854 profile_print_and_cleanup(); 1855 return 0; 1856 1857 out: 1858 profile_close_perf_events(profile_obj); 1859 if (profile_obj) 1860 profiler_bpf__destroy(profile_obj); 1861 close(profile_tgt_fd); 1862 free(profile_tgt_name); 1863 return err; 1864 } 1865 1866 #endif /* BPFTOOL_WITHOUT_SKELETONS */ 1867 1868 static int do_help(int argc, char **argv) 1869 { 1870 if (json_output) { 1871 jsonw_null(json_wtr); 1872 return 0; 1873 } 1874 1875 fprintf(stderr, 1876 "Usage: %1$s %2$s { show | list } [PROG]\n" 1877 " %1$s %2$s dump xlated PROG [{ file FILE | opcodes | visual | linum }]\n" 1878 " %1$s %2$s dump jited PROG [{ file FILE | opcodes | linum }]\n" 1879 " %1$s %2$s pin PROG FILE\n" 1880 " %1$s %2$s { load | loadall } OBJ PATH \\\n" 1881 " [type TYPE] [dev NAME] \\\n" 1882 " [map { idx IDX | name NAME } MAP]\\\n" 1883 " [pinmaps MAP_DIR]\n" 1884 " %1$s %2$s attach PROG ATTACH_TYPE [MAP]\n" 1885 " %1$s %2$s detach PROG ATTACH_TYPE [MAP]\n" 1886 " %1$s %2$s run PROG \\\n" 1887 " data_in FILE \\\n" 1888 " [data_out FILE [data_size_out L]] \\\n" 1889 " [ctx_in FILE [ctx_out FILE [ctx_size_out M]]] \\\n" 1890 " [repeat N]\n" 1891 " %1$s %2$s profile PROG [duration DURATION] METRICs\n" 1892 " %1$s %2$s tracelog\n" 1893 " %1$s %2$s help\n" 1894 "\n" 1895 " " HELP_SPEC_MAP "\n" 1896 " " HELP_SPEC_PROGRAM "\n" 1897 " TYPE := { socket | kprobe | kretprobe | classifier | action |\n" 1898 " tracepoint | raw_tracepoint | xdp | perf_event | cgroup/skb |\n" 1899 " cgroup/sock | cgroup/dev | lwt_in | lwt_out | lwt_xmit |\n" 1900 " lwt_seg6local | sockops | sk_skb | sk_msg | lirc_mode2 |\n" 1901 " sk_reuseport | flow_dissector | cgroup/sysctl |\n" 1902 " cgroup/bind4 | cgroup/bind6 | cgroup/post_bind4 |\n" 1903 " cgroup/post_bind6 | cgroup/connect4 | cgroup/connect6 |\n" 1904 " cgroup/getpeername4 | cgroup/getpeername6 |\n" 1905 " cgroup/getsockname4 | cgroup/getsockname6 | cgroup/sendmsg4 |\n" 1906 " cgroup/sendmsg6 | cgroup/recvmsg4 | cgroup/recvmsg6 |\n" 1907 " cgroup/getsockopt | cgroup/setsockopt |\n" 1908 " struct_ops | fentry | fexit | freplace }\n" 1909 " ATTACH_TYPE := { msg_verdict | stream_verdict | stream_parser |\n" 1910 " flow_dissector }\n" 1911 " METRIC := { cycles | instructions | l1d_loads | llc_misses }\n" 1912 " " HELP_SPEC_OPTIONS "\n" 1913 "", 1914 bin_name, argv[-2]); 1915 1916 return 0; 1917 } 1918 1919 static const struct cmd cmds[] = { 1920 { "show", do_show }, 1921 { "list", do_show }, 1922 { "help", do_help }, 1923 { "dump", do_dump }, 1924 { "pin", do_pin }, 1925 { "load", do_load }, 1926 { "loadall", do_loadall }, 1927 { "attach", do_attach }, 1928 { "detach", do_detach }, 1929 { "tracelog", do_tracelog }, 1930 { "run", do_run }, 1931 { "profile", do_profile }, 1932 { 0 } 1933 }; 1934 1935 int do_prog(int argc, char **argv) 1936 { 1937 return cmd_select(cmds, argc, argv, do_help); 1938 } 1939