1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <limits.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <endian.h> 25 #include <fcntl.h> 26 #include <errno.h> 27 #include <ctype.h> 28 #include <asm/unistd.h> 29 #include <linux/err.h> 30 #include <linux/kernel.h> 31 #include <linux/bpf.h> 32 #include <linux/btf.h> 33 #include <linux/filter.h> 34 #include <linux/list.h> 35 #include <linux/limits.h> 36 #include <linux/perf_event.h> 37 #include <linux/ring_buffer.h> 38 #include <linux/version.h> 39 #include <sys/epoll.h> 40 #include <sys/ioctl.h> 41 #include <sys/mman.h> 42 #include <sys/stat.h> 43 #include <sys/types.h> 44 #include <sys/vfs.h> 45 #include <sys/utsname.h> 46 #include <sys/resource.h> 47 #include <tools/libc_compat.h> 48 #include <libelf.h> 49 #include <gelf.h> 50 #include <zlib.h> 51 52 #include "libbpf.h" 53 #include "bpf.h" 54 #include "btf.h" 55 #include "str_error.h" 56 #include "libbpf_internal.h" 57 #include "hashmap.h" 58 59 /* make sure libbpf doesn't use kernel-only integer typedefs */ 60 #pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 61 62 #ifndef EM_BPF 63 #define EM_BPF 247 64 #endif 65 66 #ifndef BPF_FS_MAGIC 67 #define BPF_FS_MAGIC 0xcafe4a11 68 #endif 69 70 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 71 * compilation if user enables corresponding warning. Disable it explicitly. 72 */ 73 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 74 75 #define __printf(a, b) __attribute__((format(printf, a, b))) 76 77 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); 78 static struct bpf_program *bpf_object__find_prog_by_idx(struct bpf_object *obj, 79 int idx); 80 static const struct btf_type * 81 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id); 82 83 static int __base_pr(enum libbpf_print_level level, const char *format, 84 va_list args) 85 { 86 if (level == LIBBPF_DEBUG) 87 return 0; 88 89 return vfprintf(stderr, format, args); 90 } 91 92 static libbpf_print_fn_t __libbpf_pr = __base_pr; 93 94 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) 95 { 96 libbpf_print_fn_t old_print_fn = __libbpf_pr; 97 98 __libbpf_pr = fn; 99 return old_print_fn; 100 } 101 102 __printf(2, 3) 103 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 104 { 105 va_list args; 106 107 if (!__libbpf_pr) 108 return; 109 110 va_start(args, format); 111 __libbpf_pr(level, format, args); 112 va_end(args); 113 } 114 115 static void pr_perm_msg(int err) 116 { 117 struct rlimit limit; 118 char buf[100]; 119 120 if (err != -EPERM || geteuid() != 0) 121 return; 122 123 err = getrlimit(RLIMIT_MEMLOCK, &limit); 124 if (err) 125 return; 126 127 if (limit.rlim_cur == RLIM_INFINITY) 128 return; 129 130 if (limit.rlim_cur < 1024) 131 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); 132 else if (limit.rlim_cur < 1024*1024) 133 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); 134 else 135 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); 136 137 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", 138 buf); 139 } 140 141 #define STRERR_BUFSIZE 128 142 143 /* Copied from tools/perf/util/util.h */ 144 #ifndef zfree 145 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 146 #endif 147 148 #ifndef zclose 149 # define zclose(fd) ({ \ 150 int ___err = 0; \ 151 if ((fd) >= 0) \ 152 ___err = close((fd)); \ 153 fd = -1; \ 154 ___err; }) 155 #endif 156 157 #ifdef HAVE_LIBELF_MMAP_SUPPORT 158 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 159 #else 160 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 161 #endif 162 163 static inline __u64 ptr_to_u64(const void *ptr) 164 { 165 return (__u64) (unsigned long) ptr; 166 } 167 168 struct bpf_capabilities { 169 /* v4.14: kernel support for program & map names. */ 170 __u32 name:1; 171 /* v5.2: kernel support for global data sections. */ 172 __u32 global_data:1; 173 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 174 __u32 btf_func:1; 175 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 176 __u32 btf_datasec:1; 177 /* BPF_F_MMAPABLE is supported for arrays */ 178 __u32 array_mmap:1; 179 /* BTF_FUNC_GLOBAL is supported */ 180 __u32 btf_func_global:1; 181 /* kernel support for expected_attach_type in BPF_PROG_LOAD */ 182 __u32 exp_attach_type:1; 183 }; 184 185 enum reloc_type { 186 RELO_LD64, 187 RELO_CALL, 188 RELO_DATA, 189 RELO_EXTERN, 190 }; 191 192 struct reloc_desc { 193 enum reloc_type type; 194 int insn_idx; 195 int map_idx; 196 int sym_off; 197 }; 198 199 struct bpf_sec_def; 200 201 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec, 202 struct bpf_program *prog); 203 204 struct bpf_sec_def { 205 const char *sec; 206 size_t len; 207 enum bpf_prog_type prog_type; 208 enum bpf_attach_type expected_attach_type; 209 bool is_exp_attach_type_optional; 210 bool is_attachable; 211 bool is_attach_btf; 212 attach_fn_t attach_fn; 213 }; 214 215 /* 216 * bpf_prog should be a better name but it has been used in 217 * linux/filter.h. 218 */ 219 struct bpf_program { 220 /* Index in elf obj file, for relocation use. */ 221 int idx; 222 char *name; 223 int prog_ifindex; 224 char *section_name; 225 const struct bpf_sec_def *sec_def; 226 /* section_name with / replaced by _; makes recursive pinning 227 * in bpf_object__pin_programs easier 228 */ 229 char *pin_name; 230 struct bpf_insn *insns; 231 size_t insns_cnt, main_prog_cnt; 232 enum bpf_prog_type type; 233 234 struct reloc_desc *reloc_desc; 235 int nr_reloc; 236 int log_level; 237 238 struct { 239 int nr; 240 int *fds; 241 } instances; 242 bpf_program_prep_t preprocessor; 243 244 struct bpf_object *obj; 245 void *priv; 246 bpf_program_clear_priv_t clear_priv; 247 248 enum bpf_attach_type expected_attach_type; 249 __u32 attach_btf_id; 250 __u32 attach_prog_fd; 251 void *func_info; 252 __u32 func_info_rec_size; 253 __u32 func_info_cnt; 254 255 struct bpf_capabilities *caps; 256 257 void *line_info; 258 __u32 line_info_rec_size; 259 __u32 line_info_cnt; 260 __u32 prog_flags; 261 }; 262 263 struct bpf_struct_ops { 264 const char *tname; 265 const struct btf_type *type; 266 struct bpf_program **progs; 267 __u32 *kern_func_off; 268 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ 269 void *data; 270 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in 271 * btf_vmlinux's format. 272 * struct bpf_struct_ops_tcp_congestion_ops { 273 * [... some other kernel fields ...] 274 * struct tcp_congestion_ops data; 275 * } 276 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) 277 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" 278 * from "data". 279 */ 280 void *kern_vdata; 281 __u32 type_id; 282 }; 283 284 #define DATA_SEC ".data" 285 #define BSS_SEC ".bss" 286 #define RODATA_SEC ".rodata" 287 #define KCONFIG_SEC ".kconfig" 288 #define KSYMS_SEC ".ksyms" 289 #define STRUCT_OPS_SEC ".struct_ops" 290 291 enum libbpf_map_type { 292 LIBBPF_MAP_UNSPEC, 293 LIBBPF_MAP_DATA, 294 LIBBPF_MAP_BSS, 295 LIBBPF_MAP_RODATA, 296 LIBBPF_MAP_KCONFIG, 297 }; 298 299 static const char * const libbpf_type_to_btf_name[] = { 300 [LIBBPF_MAP_DATA] = DATA_SEC, 301 [LIBBPF_MAP_BSS] = BSS_SEC, 302 [LIBBPF_MAP_RODATA] = RODATA_SEC, 303 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC, 304 }; 305 306 struct bpf_map { 307 char *name; 308 int fd; 309 int sec_idx; 310 size_t sec_offset; 311 int map_ifindex; 312 int inner_map_fd; 313 struct bpf_map_def def; 314 __u32 numa_node; 315 __u32 btf_var_idx; 316 __u32 btf_key_type_id; 317 __u32 btf_value_type_id; 318 __u32 btf_vmlinux_value_type_id; 319 void *priv; 320 bpf_map_clear_priv_t clear_priv; 321 enum libbpf_map_type libbpf_type; 322 void *mmaped; 323 struct bpf_struct_ops *st_ops; 324 struct bpf_map *inner_map; 325 void **init_slots; 326 int init_slots_sz; 327 char *pin_path; 328 bool pinned; 329 bool reused; 330 }; 331 332 enum extern_type { 333 EXT_UNKNOWN, 334 EXT_KCFG, 335 EXT_KSYM, 336 }; 337 338 enum kcfg_type { 339 KCFG_UNKNOWN, 340 KCFG_CHAR, 341 KCFG_BOOL, 342 KCFG_INT, 343 KCFG_TRISTATE, 344 KCFG_CHAR_ARR, 345 }; 346 347 struct extern_desc { 348 enum extern_type type; 349 int sym_idx; 350 int btf_id; 351 int sec_btf_id; 352 const char *name; 353 bool is_set; 354 bool is_weak; 355 union { 356 struct { 357 enum kcfg_type type; 358 int sz; 359 int align; 360 int data_off; 361 bool is_signed; 362 } kcfg; 363 struct { 364 unsigned long long addr; 365 } ksym; 366 }; 367 }; 368 369 static LIST_HEAD(bpf_objects_list); 370 371 struct bpf_object { 372 char name[BPF_OBJ_NAME_LEN]; 373 char license[64]; 374 __u32 kern_version; 375 376 struct bpf_program *programs; 377 size_t nr_programs; 378 struct bpf_map *maps; 379 size_t nr_maps; 380 size_t maps_cap; 381 382 char *kconfig; 383 struct extern_desc *externs; 384 int nr_extern; 385 int kconfig_map_idx; 386 387 bool loaded; 388 bool has_pseudo_calls; 389 390 /* 391 * Information when doing elf related work. Only valid if fd 392 * is valid. 393 */ 394 struct { 395 int fd; 396 const void *obj_buf; 397 size_t obj_buf_sz; 398 Elf *elf; 399 GElf_Ehdr ehdr; 400 Elf_Data *symbols; 401 Elf_Data *data; 402 Elf_Data *rodata; 403 Elf_Data *bss; 404 Elf_Data *st_ops_data; 405 size_t strtabidx; 406 struct { 407 GElf_Shdr shdr; 408 Elf_Data *data; 409 } *reloc_sects; 410 int nr_reloc_sects; 411 int maps_shndx; 412 int btf_maps_shndx; 413 __u32 btf_maps_sec_btf_id; 414 int text_shndx; 415 int symbols_shndx; 416 int data_shndx; 417 int rodata_shndx; 418 int bss_shndx; 419 int st_ops_shndx; 420 } efile; 421 /* 422 * All loaded bpf_object is linked in a list, which is 423 * hidden to caller. bpf_objects__<func> handlers deal with 424 * all objects. 425 */ 426 struct list_head list; 427 428 struct btf *btf; 429 /* Parse and load BTF vmlinux if any of the programs in the object need 430 * it at load time. 431 */ 432 struct btf *btf_vmlinux; 433 struct btf_ext *btf_ext; 434 435 void *priv; 436 bpf_object_clear_priv_t clear_priv; 437 438 struct bpf_capabilities caps; 439 440 char path[]; 441 }; 442 #define obj_elf_valid(o) ((o)->efile.elf) 443 444 void bpf_program__unload(struct bpf_program *prog) 445 { 446 int i; 447 448 if (!prog) 449 return; 450 451 /* 452 * If the object is opened but the program was never loaded, 453 * it is possible that prog->instances.nr == -1. 454 */ 455 if (prog->instances.nr > 0) { 456 for (i = 0; i < prog->instances.nr; i++) 457 zclose(prog->instances.fds[i]); 458 } else if (prog->instances.nr != -1) { 459 pr_warn("Internal error: instances.nr is %d\n", 460 prog->instances.nr); 461 } 462 463 prog->instances.nr = -1; 464 zfree(&prog->instances.fds); 465 466 zfree(&prog->func_info); 467 zfree(&prog->line_info); 468 } 469 470 static void bpf_program__exit(struct bpf_program *prog) 471 { 472 if (!prog) 473 return; 474 475 if (prog->clear_priv) 476 prog->clear_priv(prog, prog->priv); 477 478 prog->priv = NULL; 479 prog->clear_priv = NULL; 480 481 bpf_program__unload(prog); 482 zfree(&prog->name); 483 zfree(&prog->section_name); 484 zfree(&prog->pin_name); 485 zfree(&prog->insns); 486 zfree(&prog->reloc_desc); 487 488 prog->nr_reloc = 0; 489 prog->insns_cnt = 0; 490 prog->idx = -1; 491 } 492 493 static char *__bpf_program__pin_name(struct bpf_program *prog) 494 { 495 char *name, *p; 496 497 name = p = strdup(prog->section_name); 498 while ((p = strchr(p, '/'))) 499 *p = '_'; 500 501 return name; 502 } 503 504 static int 505 bpf_program__init(void *data, size_t size, char *section_name, int idx, 506 struct bpf_program *prog) 507 { 508 const size_t bpf_insn_sz = sizeof(struct bpf_insn); 509 510 if (size == 0 || size % bpf_insn_sz) { 511 pr_warn("corrupted section '%s', size: %zu\n", 512 section_name, size); 513 return -EINVAL; 514 } 515 516 memset(prog, 0, sizeof(*prog)); 517 518 prog->section_name = strdup(section_name); 519 if (!prog->section_name) { 520 pr_warn("failed to alloc name for prog under section(%d) %s\n", 521 idx, section_name); 522 goto errout; 523 } 524 525 prog->pin_name = __bpf_program__pin_name(prog); 526 if (!prog->pin_name) { 527 pr_warn("failed to alloc pin name for prog under section(%d) %s\n", 528 idx, section_name); 529 goto errout; 530 } 531 532 prog->insns = malloc(size); 533 if (!prog->insns) { 534 pr_warn("failed to alloc insns for prog under section %s\n", 535 section_name); 536 goto errout; 537 } 538 prog->insns_cnt = size / bpf_insn_sz; 539 memcpy(prog->insns, data, size); 540 prog->idx = idx; 541 prog->instances.fds = NULL; 542 prog->instances.nr = -1; 543 prog->type = BPF_PROG_TYPE_UNSPEC; 544 545 return 0; 546 errout: 547 bpf_program__exit(prog); 548 return -ENOMEM; 549 } 550 551 static int 552 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 553 char *section_name, int idx) 554 { 555 struct bpf_program prog, *progs; 556 int nr_progs, err; 557 558 err = bpf_program__init(data, size, section_name, idx, &prog); 559 if (err) 560 return err; 561 562 prog.caps = &obj->caps; 563 progs = obj->programs; 564 nr_progs = obj->nr_programs; 565 566 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 567 if (!progs) { 568 /* 569 * In this case the original obj->programs 570 * is still valid, so don't need special treat for 571 * bpf_close_object(). 572 */ 573 pr_warn("failed to alloc a new program under section '%s'\n", 574 section_name); 575 bpf_program__exit(&prog); 576 return -ENOMEM; 577 } 578 579 pr_debug("found program %s\n", prog.section_name); 580 obj->programs = progs; 581 obj->nr_programs = nr_progs + 1; 582 prog.obj = obj; 583 progs[nr_progs] = prog; 584 return 0; 585 } 586 587 static int 588 bpf_object__init_prog_names(struct bpf_object *obj) 589 { 590 Elf_Data *symbols = obj->efile.symbols; 591 struct bpf_program *prog; 592 size_t pi, si; 593 594 for (pi = 0; pi < obj->nr_programs; pi++) { 595 const char *name = NULL; 596 597 prog = &obj->programs[pi]; 598 599 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 600 si++) { 601 GElf_Sym sym; 602 603 if (!gelf_getsym(symbols, si, &sym)) 604 continue; 605 if (sym.st_shndx != prog->idx) 606 continue; 607 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 608 continue; 609 610 name = elf_strptr(obj->efile.elf, 611 obj->efile.strtabidx, 612 sym.st_name); 613 if (!name) { 614 pr_warn("failed to get sym name string for prog %s\n", 615 prog->section_name); 616 return -LIBBPF_ERRNO__LIBELF; 617 } 618 } 619 620 if (!name && prog->idx == obj->efile.text_shndx) 621 name = ".text"; 622 623 if (!name) { 624 pr_warn("failed to find sym for prog %s\n", 625 prog->section_name); 626 return -EINVAL; 627 } 628 629 prog->name = strdup(name); 630 if (!prog->name) { 631 pr_warn("failed to allocate memory for prog sym %s\n", 632 name); 633 return -ENOMEM; 634 } 635 } 636 637 return 0; 638 } 639 640 static __u32 get_kernel_version(void) 641 { 642 __u32 major, minor, patch; 643 struct utsname info; 644 645 uname(&info); 646 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3) 647 return 0; 648 return KERNEL_VERSION(major, minor, patch); 649 } 650 651 static const struct btf_member * 652 find_member_by_offset(const struct btf_type *t, __u32 bit_offset) 653 { 654 struct btf_member *m; 655 int i; 656 657 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 658 if (btf_member_bit_offset(t, i) == bit_offset) 659 return m; 660 } 661 662 return NULL; 663 } 664 665 static const struct btf_member * 666 find_member_by_name(const struct btf *btf, const struct btf_type *t, 667 const char *name) 668 { 669 struct btf_member *m; 670 int i; 671 672 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 673 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) 674 return m; 675 } 676 677 return NULL; 678 } 679 680 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" 681 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 682 const char *name, __u32 kind); 683 684 static int 685 find_struct_ops_kern_types(const struct btf *btf, const char *tname, 686 const struct btf_type **type, __u32 *type_id, 687 const struct btf_type **vtype, __u32 *vtype_id, 688 const struct btf_member **data_member) 689 { 690 const struct btf_type *kern_type, *kern_vtype; 691 const struct btf_member *kern_data_member; 692 __s32 kern_vtype_id, kern_type_id; 693 __u32 i; 694 695 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT); 696 if (kern_type_id < 0) { 697 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", 698 tname); 699 return kern_type_id; 700 } 701 kern_type = btf__type_by_id(btf, kern_type_id); 702 703 /* Find the corresponding "map_value" type that will be used 704 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example, 705 * find "struct bpf_struct_ops_tcp_congestion_ops" from the 706 * btf_vmlinux. 707 */ 708 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX, 709 tname, BTF_KIND_STRUCT); 710 if (kern_vtype_id < 0) { 711 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n", 712 STRUCT_OPS_VALUE_PREFIX, tname); 713 return kern_vtype_id; 714 } 715 kern_vtype = btf__type_by_id(btf, kern_vtype_id); 716 717 /* Find "struct tcp_congestion_ops" from 718 * struct bpf_struct_ops_tcp_congestion_ops { 719 * [ ... ] 720 * struct tcp_congestion_ops data; 721 * } 722 */ 723 kern_data_member = btf_members(kern_vtype); 724 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { 725 if (kern_data_member->type == kern_type_id) 726 break; 727 } 728 if (i == btf_vlen(kern_vtype)) { 729 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n", 730 tname, STRUCT_OPS_VALUE_PREFIX, tname); 731 return -EINVAL; 732 } 733 734 *type = kern_type; 735 *type_id = kern_type_id; 736 *vtype = kern_vtype; 737 *vtype_id = kern_vtype_id; 738 *data_member = kern_data_member; 739 740 return 0; 741 } 742 743 static bool bpf_map__is_struct_ops(const struct bpf_map *map) 744 { 745 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; 746 } 747 748 /* Init the map's fields that depend on kern_btf */ 749 static int bpf_map__init_kern_struct_ops(struct bpf_map *map, 750 const struct btf *btf, 751 const struct btf *kern_btf) 752 { 753 const struct btf_member *member, *kern_member, *kern_data_member; 754 const struct btf_type *type, *kern_type, *kern_vtype; 755 __u32 i, kern_type_id, kern_vtype_id, kern_data_off; 756 struct bpf_struct_ops *st_ops; 757 void *data, *kern_data; 758 const char *tname; 759 int err; 760 761 st_ops = map->st_ops; 762 type = st_ops->type; 763 tname = st_ops->tname; 764 err = find_struct_ops_kern_types(kern_btf, tname, 765 &kern_type, &kern_type_id, 766 &kern_vtype, &kern_vtype_id, 767 &kern_data_member); 768 if (err) 769 return err; 770 771 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", 772 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); 773 774 map->def.value_size = kern_vtype->size; 775 map->btf_vmlinux_value_type_id = kern_vtype_id; 776 777 st_ops->kern_vdata = calloc(1, kern_vtype->size); 778 if (!st_ops->kern_vdata) 779 return -ENOMEM; 780 781 data = st_ops->data; 782 kern_data_off = kern_data_member->offset / 8; 783 kern_data = st_ops->kern_vdata + kern_data_off; 784 785 member = btf_members(type); 786 for (i = 0; i < btf_vlen(type); i++, member++) { 787 const struct btf_type *mtype, *kern_mtype; 788 __u32 mtype_id, kern_mtype_id; 789 void *mdata, *kern_mdata; 790 __s64 msize, kern_msize; 791 __u32 moff, kern_moff; 792 __u32 kern_member_idx; 793 const char *mname; 794 795 mname = btf__name_by_offset(btf, member->name_off); 796 kern_member = find_member_by_name(kern_btf, kern_type, mname); 797 if (!kern_member) { 798 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", 799 map->name, mname); 800 return -ENOTSUP; 801 } 802 803 kern_member_idx = kern_member - btf_members(kern_type); 804 if (btf_member_bitfield_size(type, i) || 805 btf_member_bitfield_size(kern_type, kern_member_idx)) { 806 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", 807 map->name, mname); 808 return -ENOTSUP; 809 } 810 811 moff = member->offset / 8; 812 kern_moff = kern_member->offset / 8; 813 814 mdata = data + moff; 815 kern_mdata = kern_data + kern_moff; 816 817 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); 818 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, 819 &kern_mtype_id); 820 if (BTF_INFO_KIND(mtype->info) != 821 BTF_INFO_KIND(kern_mtype->info)) { 822 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", 823 map->name, mname, BTF_INFO_KIND(mtype->info), 824 BTF_INFO_KIND(kern_mtype->info)); 825 return -ENOTSUP; 826 } 827 828 if (btf_is_ptr(mtype)) { 829 struct bpf_program *prog; 830 831 mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id); 832 kern_mtype = skip_mods_and_typedefs(kern_btf, 833 kern_mtype->type, 834 &kern_mtype_id); 835 if (!btf_is_func_proto(mtype) || 836 !btf_is_func_proto(kern_mtype)) { 837 pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n", 838 map->name, mname); 839 return -ENOTSUP; 840 } 841 842 prog = st_ops->progs[i]; 843 if (!prog) { 844 pr_debug("struct_ops init_kern %s: func ptr %s is not set\n", 845 map->name, mname); 846 continue; 847 } 848 849 prog->attach_btf_id = kern_type_id; 850 prog->expected_attach_type = kern_member_idx; 851 852 st_ops->kern_func_off[i] = kern_data_off + kern_moff; 853 854 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", 855 map->name, mname, prog->name, moff, 856 kern_moff); 857 858 continue; 859 } 860 861 msize = btf__resolve_size(btf, mtype_id); 862 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); 863 if (msize < 0 || kern_msize < 0 || msize != kern_msize) { 864 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", 865 map->name, mname, (ssize_t)msize, 866 (ssize_t)kern_msize); 867 return -ENOTSUP; 868 } 869 870 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", 871 map->name, mname, (unsigned int)msize, 872 moff, kern_moff); 873 memcpy(kern_mdata, mdata, msize); 874 } 875 876 return 0; 877 } 878 879 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) 880 { 881 struct bpf_map *map; 882 size_t i; 883 int err; 884 885 for (i = 0; i < obj->nr_maps; i++) { 886 map = &obj->maps[i]; 887 888 if (!bpf_map__is_struct_ops(map)) 889 continue; 890 891 err = bpf_map__init_kern_struct_ops(map, obj->btf, 892 obj->btf_vmlinux); 893 if (err) 894 return err; 895 } 896 897 return 0; 898 } 899 900 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj) 901 { 902 const struct btf_type *type, *datasec; 903 const struct btf_var_secinfo *vsi; 904 struct bpf_struct_ops *st_ops; 905 const char *tname, *var_name; 906 __s32 type_id, datasec_id; 907 const struct btf *btf; 908 struct bpf_map *map; 909 __u32 i; 910 911 if (obj->efile.st_ops_shndx == -1) 912 return 0; 913 914 btf = obj->btf; 915 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC, 916 BTF_KIND_DATASEC); 917 if (datasec_id < 0) { 918 pr_warn("struct_ops init: DATASEC %s not found\n", 919 STRUCT_OPS_SEC); 920 return -EINVAL; 921 } 922 923 datasec = btf__type_by_id(btf, datasec_id); 924 vsi = btf_var_secinfos(datasec); 925 for (i = 0; i < btf_vlen(datasec); i++, vsi++) { 926 type = btf__type_by_id(obj->btf, vsi->type); 927 var_name = btf__name_by_offset(obj->btf, type->name_off); 928 929 type_id = btf__resolve_type(obj->btf, vsi->type); 930 if (type_id < 0) { 931 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", 932 vsi->type, STRUCT_OPS_SEC); 933 return -EINVAL; 934 } 935 936 type = btf__type_by_id(obj->btf, type_id); 937 tname = btf__name_by_offset(obj->btf, type->name_off); 938 if (!tname[0]) { 939 pr_warn("struct_ops init: anonymous type is not supported\n"); 940 return -ENOTSUP; 941 } 942 if (!btf_is_struct(type)) { 943 pr_warn("struct_ops init: %s is not a struct\n", tname); 944 return -EINVAL; 945 } 946 947 map = bpf_object__add_map(obj); 948 if (IS_ERR(map)) 949 return PTR_ERR(map); 950 951 map->sec_idx = obj->efile.st_ops_shndx; 952 map->sec_offset = vsi->offset; 953 map->name = strdup(var_name); 954 if (!map->name) 955 return -ENOMEM; 956 957 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; 958 map->def.key_size = sizeof(int); 959 map->def.value_size = type->size; 960 map->def.max_entries = 1; 961 962 map->st_ops = calloc(1, sizeof(*map->st_ops)); 963 if (!map->st_ops) 964 return -ENOMEM; 965 st_ops = map->st_ops; 966 st_ops->data = malloc(type->size); 967 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); 968 st_ops->kern_func_off = malloc(btf_vlen(type) * 969 sizeof(*st_ops->kern_func_off)); 970 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) 971 return -ENOMEM; 972 973 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) { 974 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", 975 var_name, STRUCT_OPS_SEC); 976 return -EINVAL; 977 } 978 979 memcpy(st_ops->data, 980 obj->efile.st_ops_data->d_buf + vsi->offset, 981 type->size); 982 st_ops->tname = tname; 983 st_ops->type = type; 984 st_ops->type_id = type_id; 985 986 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", 987 tname, type_id, var_name, vsi->offset); 988 } 989 990 return 0; 991 } 992 993 static struct bpf_object *bpf_object__new(const char *path, 994 const void *obj_buf, 995 size_t obj_buf_sz, 996 const char *obj_name) 997 { 998 struct bpf_object *obj; 999 char *end; 1000 1001 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 1002 if (!obj) { 1003 pr_warn("alloc memory failed for %s\n", path); 1004 return ERR_PTR(-ENOMEM); 1005 } 1006 1007 strcpy(obj->path, path); 1008 if (obj_name) { 1009 strncpy(obj->name, obj_name, sizeof(obj->name) - 1); 1010 obj->name[sizeof(obj->name) - 1] = 0; 1011 } else { 1012 /* Using basename() GNU version which doesn't modify arg. */ 1013 strncpy(obj->name, basename((void *)path), 1014 sizeof(obj->name) - 1); 1015 end = strchr(obj->name, '.'); 1016 if (end) 1017 *end = 0; 1018 } 1019 1020 obj->efile.fd = -1; 1021 /* 1022 * Caller of this function should also call 1023 * bpf_object__elf_finish() after data collection to return 1024 * obj_buf to user. If not, we should duplicate the buffer to 1025 * avoid user freeing them before elf finish. 1026 */ 1027 obj->efile.obj_buf = obj_buf; 1028 obj->efile.obj_buf_sz = obj_buf_sz; 1029 obj->efile.maps_shndx = -1; 1030 obj->efile.btf_maps_shndx = -1; 1031 obj->efile.data_shndx = -1; 1032 obj->efile.rodata_shndx = -1; 1033 obj->efile.bss_shndx = -1; 1034 obj->efile.st_ops_shndx = -1; 1035 obj->kconfig_map_idx = -1; 1036 1037 obj->kern_version = get_kernel_version(); 1038 obj->loaded = false; 1039 1040 INIT_LIST_HEAD(&obj->list); 1041 list_add(&obj->list, &bpf_objects_list); 1042 return obj; 1043 } 1044 1045 static void bpf_object__elf_finish(struct bpf_object *obj) 1046 { 1047 if (!obj_elf_valid(obj)) 1048 return; 1049 1050 if (obj->efile.elf) { 1051 elf_end(obj->efile.elf); 1052 obj->efile.elf = NULL; 1053 } 1054 obj->efile.symbols = NULL; 1055 obj->efile.data = NULL; 1056 obj->efile.rodata = NULL; 1057 obj->efile.bss = NULL; 1058 obj->efile.st_ops_data = NULL; 1059 1060 zfree(&obj->efile.reloc_sects); 1061 obj->efile.nr_reloc_sects = 0; 1062 zclose(obj->efile.fd); 1063 obj->efile.obj_buf = NULL; 1064 obj->efile.obj_buf_sz = 0; 1065 } 1066 1067 static int bpf_object__elf_init(struct bpf_object *obj) 1068 { 1069 int err = 0; 1070 GElf_Ehdr *ep; 1071 1072 if (obj_elf_valid(obj)) { 1073 pr_warn("elf init: internal error\n"); 1074 return -LIBBPF_ERRNO__LIBELF; 1075 } 1076 1077 if (obj->efile.obj_buf_sz > 0) { 1078 /* 1079 * obj_buf should have been validated by 1080 * bpf_object__open_buffer(). 1081 */ 1082 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf, 1083 obj->efile.obj_buf_sz); 1084 } else { 1085 obj->efile.fd = open(obj->path, O_RDONLY); 1086 if (obj->efile.fd < 0) { 1087 char errmsg[STRERR_BUFSIZE], *cp; 1088 1089 err = -errno; 1090 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 1091 pr_warn("failed to open %s: %s\n", obj->path, cp); 1092 return err; 1093 } 1094 1095 obj->efile.elf = elf_begin(obj->efile.fd, 1096 LIBBPF_ELF_C_READ_MMAP, NULL); 1097 } 1098 1099 if (!obj->efile.elf) { 1100 pr_warn("failed to open %s as ELF file\n", obj->path); 1101 err = -LIBBPF_ERRNO__LIBELF; 1102 goto errout; 1103 } 1104 1105 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 1106 pr_warn("failed to get EHDR from %s\n", obj->path); 1107 err = -LIBBPF_ERRNO__FORMAT; 1108 goto errout; 1109 } 1110 ep = &obj->efile.ehdr; 1111 1112 /* Old LLVM set e_machine to EM_NONE */ 1113 if (ep->e_type != ET_REL || 1114 (ep->e_machine && ep->e_machine != EM_BPF)) { 1115 pr_warn("%s is not an eBPF object file\n", obj->path); 1116 err = -LIBBPF_ERRNO__FORMAT; 1117 goto errout; 1118 } 1119 1120 return 0; 1121 errout: 1122 bpf_object__elf_finish(obj); 1123 return err; 1124 } 1125 1126 static int bpf_object__check_endianness(struct bpf_object *obj) 1127 { 1128 #if __BYTE_ORDER == __LITTLE_ENDIAN 1129 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) 1130 return 0; 1131 #elif __BYTE_ORDER == __BIG_ENDIAN 1132 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) 1133 return 0; 1134 #else 1135 # error "Unrecognized __BYTE_ORDER__" 1136 #endif 1137 pr_warn("endianness mismatch.\n"); 1138 return -LIBBPF_ERRNO__ENDIAN; 1139 } 1140 1141 static int 1142 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 1143 { 1144 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); 1145 pr_debug("license of %s is %s\n", obj->path, obj->license); 1146 return 0; 1147 } 1148 1149 static int 1150 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 1151 { 1152 __u32 kver; 1153 1154 if (size != sizeof(kver)) { 1155 pr_warn("invalid kver section in %s\n", obj->path); 1156 return -LIBBPF_ERRNO__FORMAT; 1157 } 1158 memcpy(&kver, data, sizeof(kver)); 1159 obj->kern_version = kver; 1160 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 1161 return 0; 1162 } 1163 1164 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 1165 { 1166 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 1167 type == BPF_MAP_TYPE_HASH_OF_MAPS) 1168 return true; 1169 return false; 1170 } 1171 1172 static int bpf_object_search_section_size(const struct bpf_object *obj, 1173 const char *name, size_t *d_size) 1174 { 1175 const GElf_Ehdr *ep = &obj->efile.ehdr; 1176 Elf *elf = obj->efile.elf; 1177 Elf_Scn *scn = NULL; 1178 int idx = 0; 1179 1180 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1181 const char *sec_name; 1182 Elf_Data *data; 1183 GElf_Shdr sh; 1184 1185 idx++; 1186 if (gelf_getshdr(scn, &sh) != &sh) { 1187 pr_warn("failed to get section(%d) header from %s\n", 1188 idx, obj->path); 1189 return -EIO; 1190 } 1191 1192 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 1193 if (!sec_name) { 1194 pr_warn("failed to get section(%d) name from %s\n", 1195 idx, obj->path); 1196 return -EIO; 1197 } 1198 1199 if (strcmp(name, sec_name)) 1200 continue; 1201 1202 data = elf_getdata(scn, 0); 1203 if (!data) { 1204 pr_warn("failed to get section(%d) data from %s(%s)\n", 1205 idx, name, obj->path); 1206 return -EIO; 1207 } 1208 1209 *d_size = data->d_size; 1210 return 0; 1211 } 1212 1213 return -ENOENT; 1214 } 1215 1216 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 1217 __u32 *size) 1218 { 1219 int ret = -ENOENT; 1220 size_t d_size; 1221 1222 *size = 0; 1223 if (!name) { 1224 return -EINVAL; 1225 } else if (!strcmp(name, DATA_SEC)) { 1226 if (obj->efile.data) 1227 *size = obj->efile.data->d_size; 1228 } else if (!strcmp(name, BSS_SEC)) { 1229 if (obj->efile.bss) 1230 *size = obj->efile.bss->d_size; 1231 } else if (!strcmp(name, RODATA_SEC)) { 1232 if (obj->efile.rodata) 1233 *size = obj->efile.rodata->d_size; 1234 } else if (!strcmp(name, STRUCT_OPS_SEC)) { 1235 if (obj->efile.st_ops_data) 1236 *size = obj->efile.st_ops_data->d_size; 1237 } else { 1238 ret = bpf_object_search_section_size(obj, name, &d_size); 1239 if (!ret) 1240 *size = d_size; 1241 } 1242 1243 return *size ? 0 : ret; 1244 } 1245 1246 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 1247 __u32 *off) 1248 { 1249 Elf_Data *symbols = obj->efile.symbols; 1250 const char *sname; 1251 size_t si; 1252 1253 if (!name || !off) 1254 return -EINVAL; 1255 1256 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 1257 GElf_Sym sym; 1258 1259 if (!gelf_getsym(symbols, si, &sym)) 1260 continue; 1261 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 1262 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 1263 continue; 1264 1265 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1266 sym.st_name); 1267 if (!sname) { 1268 pr_warn("failed to get sym name string for var %s\n", 1269 name); 1270 return -EIO; 1271 } 1272 if (strcmp(name, sname) == 0) { 1273 *off = sym.st_value; 1274 return 0; 1275 } 1276 } 1277 1278 return -ENOENT; 1279 } 1280 1281 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 1282 { 1283 struct bpf_map *new_maps; 1284 size_t new_cap; 1285 int i; 1286 1287 if (obj->nr_maps < obj->maps_cap) 1288 return &obj->maps[obj->nr_maps++]; 1289 1290 new_cap = max((size_t)4, obj->maps_cap * 3 / 2); 1291 new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps)); 1292 if (!new_maps) { 1293 pr_warn("alloc maps for object failed\n"); 1294 return ERR_PTR(-ENOMEM); 1295 } 1296 1297 obj->maps_cap = new_cap; 1298 obj->maps = new_maps; 1299 1300 /* zero out new maps */ 1301 memset(obj->maps + obj->nr_maps, 0, 1302 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); 1303 /* 1304 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) 1305 * when failure (zclose won't close negative fd)). 1306 */ 1307 for (i = obj->nr_maps; i < obj->maps_cap; i++) { 1308 obj->maps[i].fd = -1; 1309 obj->maps[i].inner_map_fd = -1; 1310 } 1311 1312 return &obj->maps[obj->nr_maps++]; 1313 } 1314 1315 static size_t bpf_map_mmap_sz(const struct bpf_map *map) 1316 { 1317 long page_sz = sysconf(_SC_PAGE_SIZE); 1318 size_t map_sz; 1319 1320 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries; 1321 map_sz = roundup(map_sz, page_sz); 1322 return map_sz; 1323 } 1324 1325 static char *internal_map_name(struct bpf_object *obj, 1326 enum libbpf_map_type type) 1327 { 1328 char map_name[BPF_OBJ_NAME_LEN], *p; 1329 const char *sfx = libbpf_type_to_btf_name[type]; 1330 int sfx_len = max((size_t)7, strlen(sfx)); 1331 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, 1332 strlen(obj->name)); 1333 1334 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, 1335 sfx_len, libbpf_type_to_btf_name[type]); 1336 1337 /* sanitise map name to characters allowed by kernel */ 1338 for (p = map_name; *p && p < map_name + sizeof(map_name); p++) 1339 if (!isalnum(*p) && *p != '_' && *p != '.') 1340 *p = '_'; 1341 1342 return strdup(map_name); 1343 } 1344 1345 static int 1346 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 1347 int sec_idx, void *data, size_t data_sz) 1348 { 1349 struct bpf_map_def *def; 1350 struct bpf_map *map; 1351 int err; 1352 1353 map = bpf_object__add_map(obj); 1354 if (IS_ERR(map)) 1355 return PTR_ERR(map); 1356 1357 map->libbpf_type = type; 1358 map->sec_idx = sec_idx; 1359 map->sec_offset = 0; 1360 map->name = internal_map_name(obj, type); 1361 if (!map->name) { 1362 pr_warn("failed to alloc map name\n"); 1363 return -ENOMEM; 1364 } 1365 1366 def = &map->def; 1367 def->type = BPF_MAP_TYPE_ARRAY; 1368 def->key_size = sizeof(int); 1369 def->value_size = data_sz; 1370 def->max_entries = 1; 1371 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG 1372 ? BPF_F_RDONLY_PROG : 0; 1373 def->map_flags |= BPF_F_MMAPABLE; 1374 1375 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", 1376 map->name, map->sec_idx, map->sec_offset, def->map_flags); 1377 1378 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, 1379 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1380 if (map->mmaped == MAP_FAILED) { 1381 err = -errno; 1382 map->mmaped = NULL; 1383 pr_warn("failed to alloc map '%s' content buffer: %d\n", 1384 map->name, err); 1385 zfree(&map->name); 1386 return err; 1387 } 1388 1389 if (data) 1390 memcpy(map->mmaped, data, data_sz); 1391 1392 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 1393 return 0; 1394 } 1395 1396 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 1397 { 1398 int err; 1399 1400 /* 1401 * Populate obj->maps with libbpf internal maps. 1402 */ 1403 if (obj->efile.data_shndx >= 0) { 1404 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 1405 obj->efile.data_shndx, 1406 obj->efile.data->d_buf, 1407 obj->efile.data->d_size); 1408 if (err) 1409 return err; 1410 } 1411 if (obj->efile.rodata_shndx >= 0) { 1412 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 1413 obj->efile.rodata_shndx, 1414 obj->efile.rodata->d_buf, 1415 obj->efile.rodata->d_size); 1416 if (err) 1417 return err; 1418 } 1419 if (obj->efile.bss_shndx >= 0) { 1420 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 1421 obj->efile.bss_shndx, 1422 NULL, 1423 obj->efile.bss->d_size); 1424 if (err) 1425 return err; 1426 } 1427 return 0; 1428 } 1429 1430 1431 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, 1432 const void *name) 1433 { 1434 int i; 1435 1436 for (i = 0; i < obj->nr_extern; i++) { 1437 if (strcmp(obj->externs[i].name, name) == 0) 1438 return &obj->externs[i]; 1439 } 1440 return NULL; 1441 } 1442 1443 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, 1444 char value) 1445 { 1446 switch (ext->kcfg.type) { 1447 case KCFG_BOOL: 1448 if (value == 'm') { 1449 pr_warn("extern (kcfg) %s=%c should be tristate or char\n", 1450 ext->name, value); 1451 return -EINVAL; 1452 } 1453 *(bool *)ext_val = value == 'y' ? true : false; 1454 break; 1455 case KCFG_TRISTATE: 1456 if (value == 'y') 1457 *(enum libbpf_tristate *)ext_val = TRI_YES; 1458 else if (value == 'm') 1459 *(enum libbpf_tristate *)ext_val = TRI_MODULE; 1460 else /* value == 'n' */ 1461 *(enum libbpf_tristate *)ext_val = TRI_NO; 1462 break; 1463 case KCFG_CHAR: 1464 *(char *)ext_val = value; 1465 break; 1466 case KCFG_UNKNOWN: 1467 case KCFG_INT: 1468 case KCFG_CHAR_ARR: 1469 default: 1470 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n", 1471 ext->name, value); 1472 return -EINVAL; 1473 } 1474 ext->is_set = true; 1475 return 0; 1476 } 1477 1478 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, 1479 const char *value) 1480 { 1481 size_t len; 1482 1483 if (ext->kcfg.type != KCFG_CHAR_ARR) { 1484 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value); 1485 return -EINVAL; 1486 } 1487 1488 len = strlen(value); 1489 if (value[len - 1] != '"') { 1490 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n", 1491 ext->name, value); 1492 return -EINVAL; 1493 } 1494 1495 /* strip quotes */ 1496 len -= 2; 1497 if (len >= ext->kcfg.sz) { 1498 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n", 1499 ext->name, value, len, ext->kcfg.sz - 1); 1500 len = ext->kcfg.sz - 1; 1501 } 1502 memcpy(ext_val, value + 1, len); 1503 ext_val[len] = '\0'; 1504 ext->is_set = true; 1505 return 0; 1506 } 1507 1508 static int parse_u64(const char *value, __u64 *res) 1509 { 1510 char *value_end; 1511 int err; 1512 1513 errno = 0; 1514 *res = strtoull(value, &value_end, 0); 1515 if (errno) { 1516 err = -errno; 1517 pr_warn("failed to parse '%s' as integer: %d\n", value, err); 1518 return err; 1519 } 1520 if (*value_end) { 1521 pr_warn("failed to parse '%s' as integer completely\n", value); 1522 return -EINVAL; 1523 } 1524 return 0; 1525 } 1526 1527 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) 1528 { 1529 int bit_sz = ext->kcfg.sz * 8; 1530 1531 if (ext->kcfg.sz == 8) 1532 return true; 1533 1534 /* Validate that value stored in u64 fits in integer of `ext->sz` 1535 * bytes size without any loss of information. If the target integer 1536 * is signed, we rely on the following limits of integer type of 1537 * Y bits and subsequent transformation: 1538 * 1539 * -2^(Y-1) <= X <= 2^(Y-1) - 1 1540 * 0 <= X + 2^(Y-1) <= 2^Y - 1 1541 * 0 <= X + 2^(Y-1) < 2^Y 1542 * 1543 * For unsigned target integer, check that all the (64 - Y) bits are 1544 * zero. 1545 */ 1546 if (ext->kcfg.is_signed) 1547 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); 1548 else 1549 return (v >> bit_sz) == 0; 1550 } 1551 1552 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, 1553 __u64 value) 1554 { 1555 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { 1556 pr_warn("extern (kcfg) %s=%llu should be integer\n", 1557 ext->name, (unsigned long long)value); 1558 return -EINVAL; 1559 } 1560 if (!is_kcfg_value_in_range(ext, value)) { 1561 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n", 1562 ext->name, (unsigned long long)value, ext->kcfg.sz); 1563 return -ERANGE; 1564 } 1565 switch (ext->kcfg.sz) { 1566 case 1: *(__u8 *)ext_val = value; break; 1567 case 2: *(__u16 *)ext_val = value; break; 1568 case 4: *(__u32 *)ext_val = value; break; 1569 case 8: *(__u64 *)ext_val = value; break; 1570 default: 1571 return -EINVAL; 1572 } 1573 ext->is_set = true; 1574 return 0; 1575 } 1576 1577 static int bpf_object__process_kconfig_line(struct bpf_object *obj, 1578 char *buf, void *data) 1579 { 1580 struct extern_desc *ext; 1581 char *sep, *value; 1582 int len, err = 0; 1583 void *ext_val; 1584 __u64 num; 1585 1586 if (strncmp(buf, "CONFIG_", 7)) 1587 return 0; 1588 1589 sep = strchr(buf, '='); 1590 if (!sep) { 1591 pr_warn("failed to parse '%s': no separator\n", buf); 1592 return -EINVAL; 1593 } 1594 1595 /* Trim ending '\n' */ 1596 len = strlen(buf); 1597 if (buf[len - 1] == '\n') 1598 buf[len - 1] = '\0'; 1599 /* Split on '=' and ensure that a value is present. */ 1600 *sep = '\0'; 1601 if (!sep[1]) { 1602 *sep = '='; 1603 pr_warn("failed to parse '%s': no value\n", buf); 1604 return -EINVAL; 1605 } 1606 1607 ext = find_extern_by_name(obj, buf); 1608 if (!ext || ext->is_set) 1609 return 0; 1610 1611 ext_val = data + ext->kcfg.data_off; 1612 value = sep + 1; 1613 1614 switch (*value) { 1615 case 'y': case 'n': case 'm': 1616 err = set_kcfg_value_tri(ext, ext_val, *value); 1617 break; 1618 case '"': 1619 err = set_kcfg_value_str(ext, ext_val, value); 1620 break; 1621 default: 1622 /* assume integer */ 1623 err = parse_u64(value, &num); 1624 if (err) { 1625 pr_warn("extern (kcfg) %s=%s should be integer\n", 1626 ext->name, value); 1627 return err; 1628 } 1629 err = set_kcfg_value_num(ext, ext_val, num); 1630 break; 1631 } 1632 if (err) 1633 return err; 1634 pr_debug("extern (kcfg) %s=%s\n", ext->name, value); 1635 return 0; 1636 } 1637 1638 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) 1639 { 1640 char buf[PATH_MAX]; 1641 struct utsname uts; 1642 int len, err = 0; 1643 gzFile file; 1644 1645 uname(&uts); 1646 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); 1647 if (len < 0) 1648 return -EINVAL; 1649 else if (len >= PATH_MAX) 1650 return -ENAMETOOLONG; 1651 1652 /* gzopen also accepts uncompressed files. */ 1653 file = gzopen(buf, "r"); 1654 if (!file) 1655 file = gzopen("/proc/config.gz", "r"); 1656 1657 if (!file) { 1658 pr_warn("failed to open system Kconfig\n"); 1659 return -ENOENT; 1660 } 1661 1662 while (gzgets(file, buf, sizeof(buf))) { 1663 err = bpf_object__process_kconfig_line(obj, buf, data); 1664 if (err) { 1665 pr_warn("error parsing system Kconfig line '%s': %d\n", 1666 buf, err); 1667 goto out; 1668 } 1669 } 1670 1671 out: 1672 gzclose(file); 1673 return err; 1674 } 1675 1676 static int bpf_object__read_kconfig_mem(struct bpf_object *obj, 1677 const char *config, void *data) 1678 { 1679 char buf[PATH_MAX]; 1680 int err = 0; 1681 FILE *file; 1682 1683 file = fmemopen((void *)config, strlen(config), "r"); 1684 if (!file) { 1685 err = -errno; 1686 pr_warn("failed to open in-memory Kconfig: %d\n", err); 1687 return err; 1688 } 1689 1690 while (fgets(buf, sizeof(buf), file)) { 1691 err = bpf_object__process_kconfig_line(obj, buf, data); 1692 if (err) { 1693 pr_warn("error parsing in-memory Kconfig line '%s': %d\n", 1694 buf, err); 1695 break; 1696 } 1697 } 1698 1699 fclose(file); 1700 return err; 1701 } 1702 1703 static int bpf_object__init_kconfig_map(struct bpf_object *obj) 1704 { 1705 struct extern_desc *last_ext = NULL, *ext; 1706 size_t map_sz; 1707 int i, err; 1708 1709 for (i = 0; i < obj->nr_extern; i++) { 1710 ext = &obj->externs[i]; 1711 if (ext->type == EXT_KCFG) 1712 last_ext = ext; 1713 } 1714 1715 if (!last_ext) 1716 return 0; 1717 1718 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; 1719 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, 1720 obj->efile.symbols_shndx, 1721 NULL, map_sz); 1722 if (err) 1723 return err; 1724 1725 obj->kconfig_map_idx = obj->nr_maps - 1; 1726 1727 return 0; 1728 } 1729 1730 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 1731 { 1732 Elf_Data *symbols = obj->efile.symbols; 1733 int i, map_def_sz = 0, nr_maps = 0, nr_syms; 1734 Elf_Data *data = NULL; 1735 Elf_Scn *scn; 1736 1737 if (obj->efile.maps_shndx < 0) 1738 return 0; 1739 1740 if (!symbols) 1741 return -EINVAL; 1742 1743 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); 1744 if (scn) 1745 data = elf_getdata(scn, NULL); 1746 if (!scn || !data) { 1747 pr_warn("failed to get Elf_Data from map section %d\n", 1748 obj->efile.maps_shndx); 1749 return -EINVAL; 1750 } 1751 1752 /* 1753 * Count number of maps. Each map has a name. 1754 * Array of maps is not supported: only the first element is 1755 * considered. 1756 * 1757 * TODO: Detect array of map and report error. 1758 */ 1759 nr_syms = symbols->d_size / sizeof(GElf_Sym); 1760 for (i = 0; i < nr_syms; i++) { 1761 GElf_Sym sym; 1762 1763 if (!gelf_getsym(symbols, i, &sym)) 1764 continue; 1765 if (sym.st_shndx != obj->efile.maps_shndx) 1766 continue; 1767 nr_maps++; 1768 } 1769 /* Assume equally sized map definitions */ 1770 pr_debug("maps in %s: %d maps in %zd bytes\n", 1771 obj->path, nr_maps, data->d_size); 1772 1773 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) { 1774 pr_warn("unable to determine map definition size section %s, %d maps in %zd bytes\n", 1775 obj->path, nr_maps, data->d_size); 1776 return -EINVAL; 1777 } 1778 map_def_sz = data->d_size / nr_maps; 1779 1780 /* Fill obj->maps using data in "maps" section. */ 1781 for (i = 0; i < nr_syms; i++) { 1782 GElf_Sym sym; 1783 const char *map_name; 1784 struct bpf_map_def *def; 1785 struct bpf_map *map; 1786 1787 if (!gelf_getsym(symbols, i, &sym)) 1788 continue; 1789 if (sym.st_shndx != obj->efile.maps_shndx) 1790 continue; 1791 1792 map = bpf_object__add_map(obj); 1793 if (IS_ERR(map)) 1794 return PTR_ERR(map); 1795 1796 map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1797 sym.st_name); 1798 if (!map_name) { 1799 pr_warn("failed to get map #%d name sym string for obj %s\n", 1800 i, obj->path); 1801 return -LIBBPF_ERRNO__FORMAT; 1802 } 1803 1804 map->libbpf_type = LIBBPF_MAP_UNSPEC; 1805 map->sec_idx = sym.st_shndx; 1806 map->sec_offset = sym.st_value; 1807 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", 1808 map_name, map->sec_idx, map->sec_offset); 1809 if (sym.st_value + map_def_sz > data->d_size) { 1810 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n", 1811 obj->path, map_name); 1812 return -EINVAL; 1813 } 1814 1815 map->name = strdup(map_name); 1816 if (!map->name) { 1817 pr_warn("failed to alloc map name\n"); 1818 return -ENOMEM; 1819 } 1820 pr_debug("map %d is \"%s\"\n", i, map->name); 1821 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 1822 /* 1823 * If the definition of the map in the object file fits in 1824 * bpf_map_def, copy it. Any extra fields in our version 1825 * of bpf_map_def will default to zero as a result of the 1826 * calloc above. 1827 */ 1828 if (map_def_sz <= sizeof(struct bpf_map_def)) { 1829 memcpy(&map->def, def, map_def_sz); 1830 } else { 1831 /* 1832 * Here the map structure being read is bigger than what 1833 * we expect, truncate if the excess bits are all zero. 1834 * If they are not zero, reject this map as 1835 * incompatible. 1836 */ 1837 char *b; 1838 1839 for (b = ((char *)def) + sizeof(struct bpf_map_def); 1840 b < ((char *)def) + map_def_sz; b++) { 1841 if (*b != 0) { 1842 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n", 1843 obj->path, map_name); 1844 if (strict) 1845 return -EINVAL; 1846 } 1847 } 1848 memcpy(&map->def, def, sizeof(struct bpf_map_def)); 1849 } 1850 } 1851 return 0; 1852 } 1853 1854 static const struct btf_type * 1855 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) 1856 { 1857 const struct btf_type *t = btf__type_by_id(btf, id); 1858 1859 if (res_id) 1860 *res_id = id; 1861 1862 while (btf_is_mod(t) || btf_is_typedef(t)) { 1863 if (res_id) 1864 *res_id = t->type; 1865 t = btf__type_by_id(btf, t->type); 1866 } 1867 1868 return t; 1869 } 1870 1871 static const struct btf_type * 1872 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) 1873 { 1874 const struct btf_type *t; 1875 1876 t = skip_mods_and_typedefs(btf, id, NULL); 1877 if (!btf_is_ptr(t)) 1878 return NULL; 1879 1880 t = skip_mods_and_typedefs(btf, t->type, res_id); 1881 1882 return btf_is_func_proto(t) ? t : NULL; 1883 } 1884 1885 /* 1886 * Fetch integer attribute of BTF map definition. Such attributes are 1887 * represented using a pointer to an array, in which dimensionality of array 1888 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 1889 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 1890 * type definition, while using only sizeof(void *) space in ELF data section. 1891 */ 1892 static bool get_map_field_int(const char *map_name, const struct btf *btf, 1893 const struct btf_member *m, __u32 *res) 1894 { 1895 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); 1896 const char *name = btf__name_by_offset(btf, m->name_off); 1897 const struct btf_array *arr_info; 1898 const struct btf_type *arr_t; 1899 1900 if (!btf_is_ptr(t)) { 1901 pr_warn("map '%s': attr '%s': expected PTR, got %u.\n", 1902 map_name, name, btf_kind(t)); 1903 return false; 1904 } 1905 1906 arr_t = btf__type_by_id(btf, t->type); 1907 if (!arr_t) { 1908 pr_warn("map '%s': attr '%s': type [%u] not found.\n", 1909 map_name, name, t->type); 1910 return false; 1911 } 1912 if (!btf_is_array(arr_t)) { 1913 pr_warn("map '%s': attr '%s': expected ARRAY, got %u.\n", 1914 map_name, name, btf_kind(arr_t)); 1915 return false; 1916 } 1917 arr_info = btf_array(arr_t); 1918 *res = arr_info->nelems; 1919 return true; 1920 } 1921 1922 static int build_map_pin_path(struct bpf_map *map, const char *path) 1923 { 1924 char buf[PATH_MAX]; 1925 int err, len; 1926 1927 if (!path) 1928 path = "/sys/fs/bpf"; 1929 1930 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map)); 1931 if (len < 0) 1932 return -EINVAL; 1933 else if (len >= PATH_MAX) 1934 return -ENAMETOOLONG; 1935 1936 err = bpf_map__set_pin_path(map, buf); 1937 if (err) 1938 return err; 1939 1940 return 0; 1941 } 1942 1943 1944 static int parse_btf_map_def(struct bpf_object *obj, 1945 struct bpf_map *map, 1946 const struct btf_type *def, 1947 bool strict, bool is_inner, 1948 const char *pin_root_path) 1949 { 1950 const struct btf_type *t; 1951 const struct btf_member *m; 1952 int vlen, i; 1953 1954 vlen = btf_vlen(def); 1955 m = btf_members(def); 1956 for (i = 0; i < vlen; i++, m++) { 1957 const char *name = btf__name_by_offset(obj->btf, m->name_off); 1958 1959 if (!name) { 1960 pr_warn("map '%s': invalid field #%d.\n", map->name, i); 1961 return -EINVAL; 1962 } 1963 if (strcmp(name, "type") == 0) { 1964 if (!get_map_field_int(map->name, obj->btf, m, 1965 &map->def.type)) 1966 return -EINVAL; 1967 pr_debug("map '%s': found type = %u.\n", 1968 map->name, map->def.type); 1969 } else if (strcmp(name, "max_entries") == 0) { 1970 if (!get_map_field_int(map->name, obj->btf, m, 1971 &map->def.max_entries)) 1972 return -EINVAL; 1973 pr_debug("map '%s': found max_entries = %u.\n", 1974 map->name, map->def.max_entries); 1975 } else if (strcmp(name, "map_flags") == 0) { 1976 if (!get_map_field_int(map->name, obj->btf, m, 1977 &map->def.map_flags)) 1978 return -EINVAL; 1979 pr_debug("map '%s': found map_flags = %u.\n", 1980 map->name, map->def.map_flags); 1981 } else if (strcmp(name, "numa_node") == 0) { 1982 if (!get_map_field_int(map->name, obj->btf, m, &map->numa_node)) 1983 return -EINVAL; 1984 pr_debug("map '%s': found numa_node = %u.\n", map->name, map->numa_node); 1985 } else if (strcmp(name, "key_size") == 0) { 1986 __u32 sz; 1987 1988 if (!get_map_field_int(map->name, obj->btf, m, &sz)) 1989 return -EINVAL; 1990 pr_debug("map '%s': found key_size = %u.\n", 1991 map->name, sz); 1992 if (map->def.key_size && map->def.key_size != sz) { 1993 pr_warn("map '%s': conflicting key size %u != %u.\n", 1994 map->name, map->def.key_size, sz); 1995 return -EINVAL; 1996 } 1997 map->def.key_size = sz; 1998 } else if (strcmp(name, "key") == 0) { 1999 __s64 sz; 2000 2001 t = btf__type_by_id(obj->btf, m->type); 2002 if (!t) { 2003 pr_warn("map '%s': key type [%d] not found.\n", 2004 map->name, m->type); 2005 return -EINVAL; 2006 } 2007 if (!btf_is_ptr(t)) { 2008 pr_warn("map '%s': key spec is not PTR: %u.\n", 2009 map->name, btf_kind(t)); 2010 return -EINVAL; 2011 } 2012 sz = btf__resolve_size(obj->btf, t->type); 2013 if (sz < 0) { 2014 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", 2015 map->name, t->type, (ssize_t)sz); 2016 return sz; 2017 } 2018 pr_debug("map '%s': found key [%u], sz = %zd.\n", 2019 map->name, t->type, (ssize_t)sz); 2020 if (map->def.key_size && map->def.key_size != sz) { 2021 pr_warn("map '%s': conflicting key size %u != %zd.\n", 2022 map->name, map->def.key_size, (ssize_t)sz); 2023 return -EINVAL; 2024 } 2025 map->def.key_size = sz; 2026 map->btf_key_type_id = t->type; 2027 } else if (strcmp(name, "value_size") == 0) { 2028 __u32 sz; 2029 2030 if (!get_map_field_int(map->name, obj->btf, m, &sz)) 2031 return -EINVAL; 2032 pr_debug("map '%s': found value_size = %u.\n", 2033 map->name, sz); 2034 if (map->def.value_size && map->def.value_size != sz) { 2035 pr_warn("map '%s': conflicting value size %u != %u.\n", 2036 map->name, map->def.value_size, sz); 2037 return -EINVAL; 2038 } 2039 map->def.value_size = sz; 2040 } else if (strcmp(name, "value") == 0) { 2041 __s64 sz; 2042 2043 t = btf__type_by_id(obj->btf, m->type); 2044 if (!t) { 2045 pr_warn("map '%s': value type [%d] not found.\n", 2046 map->name, m->type); 2047 return -EINVAL; 2048 } 2049 if (!btf_is_ptr(t)) { 2050 pr_warn("map '%s': value spec is not PTR: %u.\n", 2051 map->name, btf_kind(t)); 2052 return -EINVAL; 2053 } 2054 sz = btf__resolve_size(obj->btf, t->type); 2055 if (sz < 0) { 2056 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", 2057 map->name, t->type, (ssize_t)sz); 2058 return sz; 2059 } 2060 pr_debug("map '%s': found value [%u], sz = %zd.\n", 2061 map->name, t->type, (ssize_t)sz); 2062 if (map->def.value_size && map->def.value_size != sz) { 2063 pr_warn("map '%s': conflicting value size %u != %zd.\n", 2064 map->name, map->def.value_size, (ssize_t)sz); 2065 return -EINVAL; 2066 } 2067 map->def.value_size = sz; 2068 map->btf_value_type_id = t->type; 2069 } 2070 else if (strcmp(name, "values") == 0) { 2071 int err; 2072 2073 if (is_inner) { 2074 pr_warn("map '%s': multi-level inner maps not supported.\n", 2075 map->name); 2076 return -ENOTSUP; 2077 } 2078 if (i != vlen - 1) { 2079 pr_warn("map '%s': '%s' member should be last.\n", 2080 map->name, name); 2081 return -EINVAL; 2082 } 2083 if (!bpf_map_type__is_map_in_map(map->def.type)) { 2084 pr_warn("map '%s': should be map-in-map.\n", 2085 map->name); 2086 return -ENOTSUP; 2087 } 2088 if (map->def.value_size && map->def.value_size != 4) { 2089 pr_warn("map '%s': conflicting value size %u != 4.\n", 2090 map->name, map->def.value_size); 2091 return -EINVAL; 2092 } 2093 map->def.value_size = 4; 2094 t = btf__type_by_id(obj->btf, m->type); 2095 if (!t) { 2096 pr_warn("map '%s': map-in-map inner type [%d] not found.\n", 2097 map->name, m->type); 2098 return -EINVAL; 2099 } 2100 if (!btf_is_array(t) || btf_array(t)->nelems) { 2101 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n", 2102 map->name); 2103 return -EINVAL; 2104 } 2105 t = skip_mods_and_typedefs(obj->btf, btf_array(t)->type, 2106 NULL); 2107 if (!btf_is_ptr(t)) { 2108 pr_warn("map '%s': map-in-map inner def is of unexpected kind %u.\n", 2109 map->name, btf_kind(t)); 2110 return -EINVAL; 2111 } 2112 t = skip_mods_and_typedefs(obj->btf, t->type, NULL); 2113 if (!btf_is_struct(t)) { 2114 pr_warn("map '%s': map-in-map inner def is of unexpected kind %u.\n", 2115 map->name, btf_kind(t)); 2116 return -EINVAL; 2117 } 2118 2119 map->inner_map = calloc(1, sizeof(*map->inner_map)); 2120 if (!map->inner_map) 2121 return -ENOMEM; 2122 map->inner_map->sec_idx = obj->efile.btf_maps_shndx; 2123 map->inner_map->name = malloc(strlen(map->name) + 2124 sizeof(".inner") + 1); 2125 if (!map->inner_map->name) 2126 return -ENOMEM; 2127 sprintf(map->inner_map->name, "%s.inner", map->name); 2128 2129 err = parse_btf_map_def(obj, map->inner_map, t, strict, 2130 true /* is_inner */, NULL); 2131 if (err) 2132 return err; 2133 } else if (strcmp(name, "pinning") == 0) { 2134 __u32 val; 2135 int err; 2136 2137 if (is_inner) { 2138 pr_debug("map '%s': inner def can't be pinned.\n", 2139 map->name); 2140 return -EINVAL; 2141 } 2142 if (!get_map_field_int(map->name, obj->btf, m, &val)) 2143 return -EINVAL; 2144 pr_debug("map '%s': found pinning = %u.\n", 2145 map->name, val); 2146 2147 if (val != LIBBPF_PIN_NONE && 2148 val != LIBBPF_PIN_BY_NAME) { 2149 pr_warn("map '%s': invalid pinning value %u.\n", 2150 map->name, val); 2151 return -EINVAL; 2152 } 2153 if (val == LIBBPF_PIN_BY_NAME) { 2154 err = build_map_pin_path(map, pin_root_path); 2155 if (err) { 2156 pr_warn("map '%s': couldn't build pin path.\n", 2157 map->name); 2158 return err; 2159 } 2160 } 2161 } else { 2162 if (strict) { 2163 pr_warn("map '%s': unknown field '%s'.\n", 2164 map->name, name); 2165 return -ENOTSUP; 2166 } 2167 pr_debug("map '%s': ignoring unknown field '%s'.\n", 2168 map->name, name); 2169 } 2170 } 2171 2172 if (map->def.type == BPF_MAP_TYPE_UNSPEC) { 2173 pr_warn("map '%s': map type isn't specified.\n", map->name); 2174 return -EINVAL; 2175 } 2176 2177 return 0; 2178 } 2179 2180 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 2181 const struct btf_type *sec, 2182 int var_idx, int sec_idx, 2183 const Elf_Data *data, bool strict, 2184 const char *pin_root_path) 2185 { 2186 const struct btf_type *var, *def; 2187 const struct btf_var_secinfo *vi; 2188 const struct btf_var *var_extra; 2189 const char *map_name; 2190 struct bpf_map *map; 2191 2192 vi = btf_var_secinfos(sec) + var_idx; 2193 var = btf__type_by_id(obj->btf, vi->type); 2194 var_extra = btf_var(var); 2195 map_name = btf__name_by_offset(obj->btf, var->name_off); 2196 2197 if (map_name == NULL || map_name[0] == '\0') { 2198 pr_warn("map #%d: empty name.\n", var_idx); 2199 return -EINVAL; 2200 } 2201 if ((__u64)vi->offset + vi->size > data->d_size) { 2202 pr_warn("map '%s' BTF data is corrupted.\n", map_name); 2203 return -EINVAL; 2204 } 2205 if (!btf_is_var(var)) { 2206 pr_warn("map '%s': unexpected var kind %u.\n", 2207 map_name, btf_kind(var)); 2208 return -EINVAL; 2209 } 2210 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && 2211 var_extra->linkage != BTF_VAR_STATIC) { 2212 pr_warn("map '%s': unsupported var linkage %u.\n", 2213 map_name, var_extra->linkage); 2214 return -EOPNOTSUPP; 2215 } 2216 2217 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 2218 if (!btf_is_struct(def)) { 2219 pr_warn("map '%s': unexpected def kind %u.\n", 2220 map_name, btf_kind(var)); 2221 return -EINVAL; 2222 } 2223 if (def->size > vi->size) { 2224 pr_warn("map '%s': invalid def size.\n", map_name); 2225 return -EINVAL; 2226 } 2227 2228 map = bpf_object__add_map(obj); 2229 if (IS_ERR(map)) 2230 return PTR_ERR(map); 2231 map->name = strdup(map_name); 2232 if (!map->name) { 2233 pr_warn("map '%s': failed to alloc map name.\n", map_name); 2234 return -ENOMEM; 2235 } 2236 map->libbpf_type = LIBBPF_MAP_UNSPEC; 2237 map->def.type = BPF_MAP_TYPE_UNSPEC; 2238 map->sec_idx = sec_idx; 2239 map->sec_offset = vi->offset; 2240 map->btf_var_idx = var_idx; 2241 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 2242 map_name, map->sec_idx, map->sec_offset); 2243 2244 return parse_btf_map_def(obj, map, def, strict, false, pin_root_path); 2245 } 2246 2247 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, 2248 const char *pin_root_path) 2249 { 2250 const struct btf_type *sec = NULL; 2251 int nr_types, i, vlen, err; 2252 const struct btf_type *t; 2253 const char *name; 2254 Elf_Data *data; 2255 Elf_Scn *scn; 2256 2257 if (obj->efile.btf_maps_shndx < 0) 2258 return 0; 2259 2260 scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx); 2261 if (scn) 2262 data = elf_getdata(scn, NULL); 2263 if (!scn || !data) { 2264 pr_warn("failed to get Elf_Data from map section %d (%s)\n", 2265 obj->efile.maps_shndx, MAPS_ELF_SEC); 2266 return -EINVAL; 2267 } 2268 2269 nr_types = btf__get_nr_types(obj->btf); 2270 for (i = 1; i <= nr_types; i++) { 2271 t = btf__type_by_id(obj->btf, i); 2272 if (!btf_is_datasec(t)) 2273 continue; 2274 name = btf__name_by_offset(obj->btf, t->name_off); 2275 if (strcmp(name, MAPS_ELF_SEC) == 0) { 2276 sec = t; 2277 obj->efile.btf_maps_sec_btf_id = i; 2278 break; 2279 } 2280 } 2281 2282 if (!sec) { 2283 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 2284 return -ENOENT; 2285 } 2286 2287 vlen = btf_vlen(sec); 2288 for (i = 0; i < vlen; i++) { 2289 err = bpf_object__init_user_btf_map(obj, sec, i, 2290 obj->efile.btf_maps_shndx, 2291 data, strict, 2292 pin_root_path); 2293 if (err) 2294 return err; 2295 } 2296 2297 return 0; 2298 } 2299 2300 static int bpf_object__init_maps(struct bpf_object *obj, 2301 const struct bpf_object_open_opts *opts) 2302 { 2303 const char *pin_root_path; 2304 bool strict; 2305 int err; 2306 2307 strict = !OPTS_GET(opts, relaxed_maps, false); 2308 pin_root_path = OPTS_GET(opts, pin_root_path, NULL); 2309 2310 err = bpf_object__init_user_maps(obj, strict); 2311 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path); 2312 err = err ?: bpf_object__init_global_data_maps(obj); 2313 err = err ?: bpf_object__init_kconfig_map(obj); 2314 err = err ?: bpf_object__init_struct_ops_maps(obj); 2315 if (err) 2316 return err; 2317 2318 return 0; 2319 } 2320 2321 static bool section_have_execinstr(struct bpf_object *obj, int idx) 2322 { 2323 Elf_Scn *scn; 2324 GElf_Shdr sh; 2325 2326 scn = elf_getscn(obj->efile.elf, idx); 2327 if (!scn) 2328 return false; 2329 2330 if (gelf_getshdr(scn, &sh) != &sh) 2331 return false; 2332 2333 if (sh.sh_flags & SHF_EXECINSTR) 2334 return true; 2335 2336 return false; 2337 } 2338 2339 static void bpf_object__sanitize_btf(struct bpf_object *obj) 2340 { 2341 bool has_func_global = obj->caps.btf_func_global; 2342 bool has_datasec = obj->caps.btf_datasec; 2343 bool has_func = obj->caps.btf_func; 2344 struct btf *btf = obj->btf; 2345 struct btf_type *t; 2346 int i, j, vlen; 2347 2348 if (!obj->btf || (has_func && has_datasec && has_func_global)) 2349 return; 2350 2351 for (i = 1; i <= btf__get_nr_types(btf); i++) { 2352 t = (struct btf_type *)btf__type_by_id(btf, i); 2353 2354 if (!has_datasec && btf_is_var(t)) { 2355 /* replace VAR with INT */ 2356 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 2357 /* 2358 * using size = 1 is the safest choice, 4 will be too 2359 * big and cause kernel BTF validation failure if 2360 * original variable took less than 4 bytes 2361 */ 2362 t->size = 1; 2363 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); 2364 } else if (!has_datasec && btf_is_datasec(t)) { 2365 /* replace DATASEC with STRUCT */ 2366 const struct btf_var_secinfo *v = btf_var_secinfos(t); 2367 struct btf_member *m = btf_members(t); 2368 struct btf_type *vt; 2369 char *name; 2370 2371 name = (char *)btf__name_by_offset(btf, t->name_off); 2372 while (*name) { 2373 if (*name == '.') 2374 *name = '_'; 2375 name++; 2376 } 2377 2378 vlen = btf_vlen(t); 2379 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 2380 for (j = 0; j < vlen; j++, v++, m++) { 2381 /* order of field assignments is important */ 2382 m->offset = v->offset * 8; 2383 m->type = v->type; 2384 /* preserve variable name as member name */ 2385 vt = (void *)btf__type_by_id(btf, v->type); 2386 m->name_off = vt->name_off; 2387 } 2388 } else if (!has_func && btf_is_func_proto(t)) { 2389 /* replace FUNC_PROTO with ENUM */ 2390 vlen = btf_vlen(t); 2391 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 2392 t->size = sizeof(__u32); /* kernel enforced */ 2393 } else if (!has_func && btf_is_func(t)) { 2394 /* replace FUNC with TYPEDEF */ 2395 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 2396 } else if (!has_func_global && btf_is_func(t)) { 2397 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ 2398 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); 2399 } 2400 } 2401 } 2402 2403 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) 2404 { 2405 if (!obj->btf_ext) 2406 return; 2407 2408 if (!obj->caps.btf_func) { 2409 btf_ext__free(obj->btf_ext); 2410 obj->btf_ext = NULL; 2411 } 2412 } 2413 2414 static bool libbpf_needs_btf(const struct bpf_object *obj) 2415 { 2416 return obj->efile.btf_maps_shndx >= 0 || 2417 obj->efile.st_ops_shndx >= 0 || 2418 obj->nr_extern > 0; 2419 } 2420 2421 static bool kernel_needs_btf(const struct bpf_object *obj) 2422 { 2423 return obj->efile.st_ops_shndx >= 0; 2424 } 2425 2426 static int bpf_object__init_btf(struct bpf_object *obj, 2427 Elf_Data *btf_data, 2428 Elf_Data *btf_ext_data) 2429 { 2430 int err = -ENOENT; 2431 2432 if (btf_data) { 2433 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 2434 if (IS_ERR(obj->btf)) { 2435 err = PTR_ERR(obj->btf); 2436 obj->btf = NULL; 2437 pr_warn("Error loading ELF section %s: %d.\n", 2438 BTF_ELF_SEC, err); 2439 goto out; 2440 } 2441 err = 0; 2442 } 2443 if (btf_ext_data) { 2444 if (!obj->btf) { 2445 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 2446 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 2447 goto out; 2448 } 2449 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 2450 btf_ext_data->d_size); 2451 if (IS_ERR(obj->btf_ext)) { 2452 pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n", 2453 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); 2454 obj->btf_ext = NULL; 2455 goto out; 2456 } 2457 } 2458 out: 2459 if (err && libbpf_needs_btf(obj)) { 2460 pr_warn("BTF is required, but is missing or corrupted.\n"); 2461 return err; 2462 } 2463 return 0; 2464 } 2465 2466 static int bpf_object__finalize_btf(struct bpf_object *obj) 2467 { 2468 int err; 2469 2470 if (!obj->btf) 2471 return 0; 2472 2473 err = btf__finalize_data(obj, obj->btf); 2474 if (!err) 2475 return 0; 2476 2477 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err); 2478 btf__free(obj->btf); 2479 obj->btf = NULL; 2480 btf_ext__free(obj->btf_ext); 2481 obj->btf_ext = NULL; 2482 2483 if (libbpf_needs_btf(obj)) { 2484 pr_warn("BTF is required, but is missing or corrupted.\n"); 2485 return -ENOENT; 2486 } 2487 return 0; 2488 } 2489 2490 static inline bool libbpf_prog_needs_vmlinux_btf(struct bpf_program *prog) 2491 { 2492 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 2493 prog->type == BPF_PROG_TYPE_LSM) 2494 return true; 2495 2496 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs 2497 * also need vmlinux BTF 2498 */ 2499 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) 2500 return true; 2501 2502 return false; 2503 } 2504 2505 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj) 2506 { 2507 struct bpf_program *prog; 2508 int err; 2509 2510 bpf_object__for_each_program(prog, obj) { 2511 if (libbpf_prog_needs_vmlinux_btf(prog)) { 2512 obj->btf_vmlinux = libbpf_find_kernel_btf(); 2513 if (IS_ERR(obj->btf_vmlinux)) { 2514 err = PTR_ERR(obj->btf_vmlinux); 2515 pr_warn("Error loading vmlinux BTF: %d\n", err); 2516 obj->btf_vmlinux = NULL; 2517 return err; 2518 } 2519 return 0; 2520 } 2521 } 2522 2523 return 0; 2524 } 2525 2526 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 2527 { 2528 int err = 0; 2529 2530 if (!obj->btf) 2531 return 0; 2532 2533 bpf_object__sanitize_btf(obj); 2534 bpf_object__sanitize_btf_ext(obj); 2535 2536 err = btf__load(obj->btf); 2537 if (err) { 2538 pr_warn("Error loading %s into kernel: %d.\n", 2539 BTF_ELF_SEC, err); 2540 btf__free(obj->btf); 2541 obj->btf = NULL; 2542 /* btf_ext can't exist without btf, so free it as well */ 2543 if (obj->btf_ext) { 2544 btf_ext__free(obj->btf_ext); 2545 obj->btf_ext = NULL; 2546 } 2547 2548 if (kernel_needs_btf(obj)) 2549 return err; 2550 } 2551 return 0; 2552 } 2553 2554 static int bpf_object__elf_collect(struct bpf_object *obj) 2555 { 2556 Elf *elf = obj->efile.elf; 2557 GElf_Ehdr *ep = &obj->efile.ehdr; 2558 Elf_Data *btf_ext_data = NULL; 2559 Elf_Data *btf_data = NULL; 2560 Elf_Scn *scn = NULL; 2561 int idx = 0, err = 0; 2562 2563 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 2564 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 2565 pr_warn("failed to get e_shstrndx from %s\n", obj->path); 2566 return -LIBBPF_ERRNO__FORMAT; 2567 } 2568 2569 while ((scn = elf_nextscn(elf, scn)) != NULL) { 2570 char *name; 2571 GElf_Shdr sh; 2572 Elf_Data *data; 2573 2574 idx++; 2575 if (gelf_getshdr(scn, &sh) != &sh) { 2576 pr_warn("failed to get section(%d) header from %s\n", 2577 idx, obj->path); 2578 return -LIBBPF_ERRNO__FORMAT; 2579 } 2580 2581 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 2582 if (!name) { 2583 pr_warn("failed to get section(%d) name from %s\n", 2584 idx, obj->path); 2585 return -LIBBPF_ERRNO__FORMAT; 2586 } 2587 2588 data = elf_getdata(scn, 0); 2589 if (!data) { 2590 pr_warn("failed to get section(%d) data from %s(%s)\n", 2591 idx, name, obj->path); 2592 return -LIBBPF_ERRNO__FORMAT; 2593 } 2594 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 2595 idx, name, (unsigned long)data->d_size, 2596 (int)sh.sh_link, (unsigned long)sh.sh_flags, 2597 (int)sh.sh_type); 2598 2599 if (strcmp(name, "license") == 0) { 2600 err = bpf_object__init_license(obj, 2601 data->d_buf, 2602 data->d_size); 2603 if (err) 2604 return err; 2605 } else if (strcmp(name, "version") == 0) { 2606 err = bpf_object__init_kversion(obj, 2607 data->d_buf, 2608 data->d_size); 2609 if (err) 2610 return err; 2611 } else if (strcmp(name, "maps") == 0) { 2612 obj->efile.maps_shndx = idx; 2613 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 2614 obj->efile.btf_maps_shndx = idx; 2615 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 2616 btf_data = data; 2617 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 2618 btf_ext_data = data; 2619 } else if (sh.sh_type == SHT_SYMTAB) { 2620 if (obj->efile.symbols) { 2621 pr_warn("bpf: multiple SYMTAB in %s\n", 2622 obj->path); 2623 return -LIBBPF_ERRNO__FORMAT; 2624 } 2625 obj->efile.symbols = data; 2626 obj->efile.symbols_shndx = idx; 2627 obj->efile.strtabidx = sh.sh_link; 2628 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 2629 if (sh.sh_flags & SHF_EXECINSTR) { 2630 if (strcmp(name, ".text") == 0) 2631 obj->efile.text_shndx = idx; 2632 err = bpf_object__add_program(obj, data->d_buf, 2633 data->d_size, 2634 name, idx); 2635 if (err) { 2636 char errmsg[STRERR_BUFSIZE]; 2637 char *cp; 2638 2639 cp = libbpf_strerror_r(-err, errmsg, 2640 sizeof(errmsg)); 2641 pr_warn("failed to alloc program %s (%s): %s", 2642 name, obj->path, cp); 2643 return err; 2644 } 2645 } else if (strcmp(name, DATA_SEC) == 0) { 2646 obj->efile.data = data; 2647 obj->efile.data_shndx = idx; 2648 } else if (strcmp(name, RODATA_SEC) == 0) { 2649 obj->efile.rodata = data; 2650 obj->efile.rodata_shndx = idx; 2651 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) { 2652 obj->efile.st_ops_data = data; 2653 obj->efile.st_ops_shndx = idx; 2654 } else { 2655 pr_debug("skip section(%d) %s\n", idx, name); 2656 } 2657 } else if (sh.sh_type == SHT_REL) { 2658 int nr_sects = obj->efile.nr_reloc_sects; 2659 void *sects = obj->efile.reloc_sects; 2660 int sec = sh.sh_info; /* points to other section */ 2661 2662 /* Only do relo for section with exec instructions */ 2663 if (!section_have_execinstr(obj, sec) && 2664 strcmp(name, ".rel" STRUCT_OPS_SEC) && 2665 strcmp(name, ".rel" MAPS_ELF_SEC)) { 2666 pr_debug("skip relo %s(%d) for section(%d)\n", 2667 name, idx, sec); 2668 continue; 2669 } 2670 2671 sects = reallocarray(sects, nr_sects + 1, 2672 sizeof(*obj->efile.reloc_sects)); 2673 if (!sects) { 2674 pr_warn("reloc_sects realloc failed\n"); 2675 return -ENOMEM; 2676 } 2677 2678 obj->efile.reloc_sects = sects; 2679 obj->efile.nr_reloc_sects++; 2680 2681 obj->efile.reloc_sects[nr_sects].shdr = sh; 2682 obj->efile.reloc_sects[nr_sects].data = data; 2683 } else if (sh.sh_type == SHT_NOBITS && 2684 strcmp(name, BSS_SEC) == 0) { 2685 obj->efile.bss = data; 2686 obj->efile.bss_shndx = idx; 2687 } else { 2688 pr_debug("skip section(%d) %s\n", idx, name); 2689 } 2690 } 2691 2692 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { 2693 pr_warn("Corrupted ELF file: index of strtab invalid\n"); 2694 return -LIBBPF_ERRNO__FORMAT; 2695 } 2696 return bpf_object__init_btf(obj, btf_data, btf_ext_data); 2697 } 2698 2699 static bool sym_is_extern(const GElf_Sym *sym) 2700 { 2701 int bind = GELF_ST_BIND(sym->st_info); 2702 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ 2703 return sym->st_shndx == SHN_UNDEF && 2704 (bind == STB_GLOBAL || bind == STB_WEAK) && 2705 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE; 2706 } 2707 2708 static int find_extern_btf_id(const struct btf *btf, const char *ext_name) 2709 { 2710 const struct btf_type *t; 2711 const char *var_name; 2712 int i, n; 2713 2714 if (!btf) 2715 return -ESRCH; 2716 2717 n = btf__get_nr_types(btf); 2718 for (i = 1; i <= n; i++) { 2719 t = btf__type_by_id(btf, i); 2720 2721 if (!btf_is_var(t)) 2722 continue; 2723 2724 var_name = btf__name_by_offset(btf, t->name_off); 2725 if (strcmp(var_name, ext_name)) 2726 continue; 2727 2728 if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) 2729 return -EINVAL; 2730 2731 return i; 2732 } 2733 2734 return -ENOENT; 2735 } 2736 2737 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) { 2738 const struct btf_var_secinfo *vs; 2739 const struct btf_type *t; 2740 int i, j, n; 2741 2742 if (!btf) 2743 return -ESRCH; 2744 2745 n = btf__get_nr_types(btf); 2746 for (i = 1; i <= n; i++) { 2747 t = btf__type_by_id(btf, i); 2748 2749 if (!btf_is_datasec(t)) 2750 continue; 2751 2752 vs = btf_var_secinfos(t); 2753 for (j = 0; j < btf_vlen(t); j++, vs++) { 2754 if (vs->type == ext_btf_id) 2755 return i; 2756 } 2757 } 2758 2759 return -ENOENT; 2760 } 2761 2762 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, 2763 bool *is_signed) 2764 { 2765 const struct btf_type *t; 2766 const char *name; 2767 2768 t = skip_mods_and_typedefs(btf, id, NULL); 2769 name = btf__name_by_offset(btf, t->name_off); 2770 2771 if (is_signed) 2772 *is_signed = false; 2773 switch (btf_kind(t)) { 2774 case BTF_KIND_INT: { 2775 int enc = btf_int_encoding(t); 2776 2777 if (enc & BTF_INT_BOOL) 2778 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; 2779 if (is_signed) 2780 *is_signed = enc & BTF_INT_SIGNED; 2781 if (t->size == 1) 2782 return KCFG_CHAR; 2783 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) 2784 return KCFG_UNKNOWN; 2785 return KCFG_INT; 2786 } 2787 case BTF_KIND_ENUM: 2788 if (t->size != 4) 2789 return KCFG_UNKNOWN; 2790 if (strcmp(name, "libbpf_tristate")) 2791 return KCFG_UNKNOWN; 2792 return KCFG_TRISTATE; 2793 case BTF_KIND_ARRAY: 2794 if (btf_array(t)->nelems == 0) 2795 return KCFG_UNKNOWN; 2796 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) 2797 return KCFG_UNKNOWN; 2798 return KCFG_CHAR_ARR; 2799 default: 2800 return KCFG_UNKNOWN; 2801 } 2802 } 2803 2804 static int cmp_externs(const void *_a, const void *_b) 2805 { 2806 const struct extern_desc *a = _a; 2807 const struct extern_desc *b = _b; 2808 2809 if (a->type != b->type) 2810 return a->type < b->type ? -1 : 1; 2811 2812 if (a->type == EXT_KCFG) { 2813 /* descending order by alignment requirements */ 2814 if (a->kcfg.align != b->kcfg.align) 2815 return a->kcfg.align > b->kcfg.align ? -1 : 1; 2816 /* ascending order by size, within same alignment class */ 2817 if (a->kcfg.sz != b->kcfg.sz) 2818 return a->kcfg.sz < b->kcfg.sz ? -1 : 1; 2819 } 2820 2821 /* resolve ties by name */ 2822 return strcmp(a->name, b->name); 2823 } 2824 2825 static int find_int_btf_id(const struct btf *btf) 2826 { 2827 const struct btf_type *t; 2828 int i, n; 2829 2830 n = btf__get_nr_types(btf); 2831 for (i = 1; i <= n; i++) { 2832 t = btf__type_by_id(btf, i); 2833 2834 if (btf_is_int(t) && btf_int_bits(t) == 32) 2835 return i; 2836 } 2837 2838 return 0; 2839 } 2840 2841 static int bpf_object__collect_externs(struct bpf_object *obj) 2842 { 2843 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL; 2844 const struct btf_type *t; 2845 struct extern_desc *ext; 2846 int i, n, off; 2847 const char *ext_name, *sec_name; 2848 Elf_Scn *scn; 2849 GElf_Shdr sh; 2850 2851 if (!obj->efile.symbols) 2852 return 0; 2853 2854 scn = elf_getscn(obj->efile.elf, obj->efile.symbols_shndx); 2855 if (!scn) 2856 return -LIBBPF_ERRNO__FORMAT; 2857 if (gelf_getshdr(scn, &sh) != &sh) 2858 return -LIBBPF_ERRNO__FORMAT; 2859 n = sh.sh_size / sh.sh_entsize; 2860 2861 pr_debug("looking for externs among %d symbols...\n", n); 2862 for (i = 0; i < n; i++) { 2863 GElf_Sym sym; 2864 2865 if (!gelf_getsym(obj->efile.symbols, i, &sym)) 2866 return -LIBBPF_ERRNO__FORMAT; 2867 if (!sym_is_extern(&sym)) 2868 continue; 2869 ext_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 2870 sym.st_name); 2871 if (!ext_name || !ext_name[0]) 2872 continue; 2873 2874 ext = obj->externs; 2875 ext = reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); 2876 if (!ext) 2877 return -ENOMEM; 2878 obj->externs = ext; 2879 ext = &ext[obj->nr_extern]; 2880 memset(ext, 0, sizeof(*ext)); 2881 obj->nr_extern++; 2882 2883 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); 2884 if (ext->btf_id <= 0) { 2885 pr_warn("failed to find BTF for extern '%s': %d\n", 2886 ext_name, ext->btf_id); 2887 return ext->btf_id; 2888 } 2889 t = btf__type_by_id(obj->btf, ext->btf_id); 2890 ext->name = btf__name_by_offset(obj->btf, t->name_off); 2891 ext->sym_idx = i; 2892 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK; 2893 2894 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); 2895 if (ext->sec_btf_id <= 0) { 2896 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n", 2897 ext_name, ext->btf_id, ext->sec_btf_id); 2898 return ext->sec_btf_id; 2899 } 2900 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); 2901 sec_name = btf__name_by_offset(obj->btf, sec->name_off); 2902 2903 if (strcmp(sec_name, KCONFIG_SEC) == 0) { 2904 kcfg_sec = sec; 2905 ext->type = EXT_KCFG; 2906 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); 2907 if (ext->kcfg.sz <= 0) { 2908 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n", 2909 ext_name, ext->kcfg.sz); 2910 return ext->kcfg.sz; 2911 } 2912 ext->kcfg.align = btf__align_of(obj->btf, t->type); 2913 if (ext->kcfg.align <= 0) { 2914 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n", 2915 ext_name, ext->kcfg.align); 2916 return -EINVAL; 2917 } 2918 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, 2919 &ext->kcfg.is_signed); 2920 if (ext->kcfg.type == KCFG_UNKNOWN) { 2921 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name); 2922 return -ENOTSUP; 2923 } 2924 } else if (strcmp(sec_name, KSYMS_SEC) == 0) { 2925 const struct btf_type *vt; 2926 2927 ksym_sec = sec; 2928 ext->type = EXT_KSYM; 2929 2930 vt = skip_mods_and_typedefs(obj->btf, t->type, NULL); 2931 if (!btf_is_void(vt)) { 2932 pr_warn("extern (ksym) '%s' is not typeless (void)\n", ext_name); 2933 return -ENOTSUP; 2934 } 2935 } else { 2936 pr_warn("unrecognized extern section '%s'\n", sec_name); 2937 return -ENOTSUP; 2938 } 2939 } 2940 pr_debug("collected %d externs total\n", obj->nr_extern); 2941 2942 if (!obj->nr_extern) 2943 return 0; 2944 2945 /* sort externs by type, for kcfg ones also by (align, size, name) */ 2946 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); 2947 2948 /* for .ksyms section, we need to turn all externs into allocated 2949 * variables in BTF to pass kernel verification; we do this by 2950 * pretending that each extern is a 8-byte variable 2951 */ 2952 if (ksym_sec) { 2953 /* find existing 4-byte integer type in BTF to use for fake 2954 * extern variables in DATASEC 2955 */ 2956 int int_btf_id = find_int_btf_id(obj->btf); 2957 2958 for (i = 0; i < obj->nr_extern; i++) { 2959 ext = &obj->externs[i]; 2960 if (ext->type != EXT_KSYM) 2961 continue; 2962 pr_debug("extern (ksym) #%d: symbol %d, name %s\n", 2963 i, ext->sym_idx, ext->name); 2964 } 2965 2966 sec = ksym_sec; 2967 n = btf_vlen(sec); 2968 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) { 2969 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 2970 struct btf_type *vt; 2971 2972 vt = (void *)btf__type_by_id(obj->btf, vs->type); 2973 ext_name = btf__name_by_offset(obj->btf, vt->name_off); 2974 ext = find_extern_by_name(obj, ext_name); 2975 if (!ext) { 2976 pr_warn("failed to find extern definition for BTF var '%s'\n", 2977 ext_name); 2978 return -ESRCH; 2979 } 2980 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 2981 vt->type = int_btf_id; 2982 vs->offset = off; 2983 vs->size = sizeof(int); 2984 } 2985 sec->size = off; 2986 } 2987 2988 if (kcfg_sec) { 2989 sec = kcfg_sec; 2990 /* for kcfg externs calculate their offsets within a .kconfig map */ 2991 off = 0; 2992 for (i = 0; i < obj->nr_extern; i++) { 2993 ext = &obj->externs[i]; 2994 if (ext->type != EXT_KCFG) 2995 continue; 2996 2997 ext->kcfg.data_off = roundup(off, ext->kcfg.align); 2998 off = ext->kcfg.data_off + ext->kcfg.sz; 2999 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", 3000 i, ext->sym_idx, ext->kcfg.data_off, ext->name); 3001 } 3002 sec->size = off; 3003 n = btf_vlen(sec); 3004 for (i = 0; i < n; i++) { 3005 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 3006 3007 t = btf__type_by_id(obj->btf, vs->type); 3008 ext_name = btf__name_by_offset(obj->btf, t->name_off); 3009 ext = find_extern_by_name(obj, ext_name); 3010 if (!ext) { 3011 pr_warn("failed to find extern definition for BTF var '%s'\n", 3012 ext_name); 3013 return -ESRCH; 3014 } 3015 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 3016 vs->offset = ext->kcfg.data_off; 3017 } 3018 } 3019 return 0; 3020 } 3021 3022 static struct bpf_program * 3023 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 3024 { 3025 struct bpf_program *prog; 3026 size_t i; 3027 3028 for (i = 0; i < obj->nr_programs; i++) { 3029 prog = &obj->programs[i]; 3030 if (prog->idx == idx) 3031 return prog; 3032 } 3033 return NULL; 3034 } 3035 3036 struct bpf_program * 3037 bpf_object__find_program_by_title(const struct bpf_object *obj, 3038 const char *title) 3039 { 3040 struct bpf_program *pos; 3041 3042 bpf_object__for_each_program(pos, obj) { 3043 if (pos->section_name && !strcmp(pos->section_name, title)) 3044 return pos; 3045 } 3046 return NULL; 3047 } 3048 3049 struct bpf_program * 3050 bpf_object__find_program_by_name(const struct bpf_object *obj, 3051 const char *name) 3052 { 3053 struct bpf_program *prog; 3054 3055 bpf_object__for_each_program(prog, obj) { 3056 if (!strcmp(prog->name, name)) 3057 return prog; 3058 } 3059 return NULL; 3060 } 3061 3062 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 3063 int shndx) 3064 { 3065 return shndx == obj->efile.data_shndx || 3066 shndx == obj->efile.bss_shndx || 3067 shndx == obj->efile.rodata_shndx; 3068 } 3069 3070 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 3071 int shndx) 3072 { 3073 return shndx == obj->efile.maps_shndx || 3074 shndx == obj->efile.btf_maps_shndx; 3075 } 3076 3077 static enum libbpf_map_type 3078 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 3079 { 3080 if (shndx == obj->efile.data_shndx) 3081 return LIBBPF_MAP_DATA; 3082 else if (shndx == obj->efile.bss_shndx) 3083 return LIBBPF_MAP_BSS; 3084 else if (shndx == obj->efile.rodata_shndx) 3085 return LIBBPF_MAP_RODATA; 3086 else if (shndx == obj->efile.symbols_shndx) 3087 return LIBBPF_MAP_KCONFIG; 3088 else 3089 return LIBBPF_MAP_UNSPEC; 3090 } 3091 3092 static int bpf_program__record_reloc(struct bpf_program *prog, 3093 struct reloc_desc *reloc_desc, 3094 __u32 insn_idx, const char *name, 3095 const GElf_Sym *sym, const GElf_Rel *rel) 3096 { 3097 struct bpf_insn *insn = &prog->insns[insn_idx]; 3098 size_t map_idx, nr_maps = prog->obj->nr_maps; 3099 struct bpf_object *obj = prog->obj; 3100 __u32 shdr_idx = sym->st_shndx; 3101 enum libbpf_map_type type; 3102 struct bpf_map *map; 3103 3104 /* sub-program call relocation */ 3105 if (insn->code == (BPF_JMP | BPF_CALL)) { 3106 if (insn->src_reg != BPF_PSEUDO_CALL) { 3107 pr_warn("incorrect bpf_call opcode\n"); 3108 return -LIBBPF_ERRNO__RELOC; 3109 } 3110 /* text_shndx can be 0, if no default "main" program exists */ 3111 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { 3112 pr_warn("bad call relo against section %u\n", shdr_idx); 3113 return -LIBBPF_ERRNO__RELOC; 3114 } 3115 if (sym->st_value % 8) { 3116 pr_warn("bad call relo offset: %zu\n", 3117 (size_t)sym->st_value); 3118 return -LIBBPF_ERRNO__RELOC; 3119 } 3120 reloc_desc->type = RELO_CALL; 3121 reloc_desc->insn_idx = insn_idx; 3122 reloc_desc->sym_off = sym->st_value; 3123 obj->has_pseudo_calls = true; 3124 return 0; 3125 } 3126 3127 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) { 3128 pr_warn("invalid relo for insns[%d].code 0x%x\n", 3129 insn_idx, insn->code); 3130 return -LIBBPF_ERRNO__RELOC; 3131 } 3132 3133 if (sym_is_extern(sym)) { 3134 int sym_idx = GELF_R_SYM(rel->r_info); 3135 int i, n = obj->nr_extern; 3136 struct extern_desc *ext; 3137 3138 for (i = 0; i < n; i++) { 3139 ext = &obj->externs[i]; 3140 if (ext->sym_idx == sym_idx) 3141 break; 3142 } 3143 if (i >= n) { 3144 pr_warn("extern relo failed to find extern for sym %d\n", 3145 sym_idx); 3146 return -LIBBPF_ERRNO__RELOC; 3147 } 3148 pr_debug("found extern #%d '%s' (sym %d) for insn %u\n", 3149 i, ext->name, ext->sym_idx, insn_idx); 3150 reloc_desc->type = RELO_EXTERN; 3151 reloc_desc->insn_idx = insn_idx; 3152 reloc_desc->sym_off = i; /* sym_off stores extern index */ 3153 return 0; 3154 } 3155 3156 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { 3157 pr_warn("invalid relo for \'%s\' in special section 0x%x; forgot to initialize global var?..\n", 3158 name, shdr_idx); 3159 return -LIBBPF_ERRNO__RELOC; 3160 } 3161 3162 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 3163 3164 /* generic map reference relocation */ 3165 if (type == LIBBPF_MAP_UNSPEC) { 3166 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { 3167 pr_warn("bad map relo against section %u\n", 3168 shdr_idx); 3169 return -LIBBPF_ERRNO__RELOC; 3170 } 3171 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3172 map = &obj->maps[map_idx]; 3173 if (map->libbpf_type != type || 3174 map->sec_idx != sym->st_shndx || 3175 map->sec_offset != sym->st_value) 3176 continue; 3177 pr_debug("found map %zd (%s, sec %d, off %zu) for insn %u\n", 3178 map_idx, map->name, map->sec_idx, 3179 map->sec_offset, insn_idx); 3180 break; 3181 } 3182 if (map_idx >= nr_maps) { 3183 pr_warn("map relo failed to find map for sec %u, off %zu\n", 3184 shdr_idx, (size_t)sym->st_value); 3185 return -LIBBPF_ERRNO__RELOC; 3186 } 3187 reloc_desc->type = RELO_LD64; 3188 reloc_desc->insn_idx = insn_idx; 3189 reloc_desc->map_idx = map_idx; 3190 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ 3191 return 0; 3192 } 3193 3194 /* global data map relocation */ 3195 if (!bpf_object__shndx_is_data(obj, shdr_idx)) { 3196 pr_warn("bad data relo against section %u\n", shdr_idx); 3197 return -LIBBPF_ERRNO__RELOC; 3198 } 3199 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 3200 map = &obj->maps[map_idx]; 3201 if (map->libbpf_type != type) 3202 continue; 3203 pr_debug("found data map %zd (%s, sec %d, off %zu) for insn %u\n", 3204 map_idx, map->name, map->sec_idx, map->sec_offset, 3205 insn_idx); 3206 break; 3207 } 3208 if (map_idx >= nr_maps) { 3209 pr_warn("data relo failed to find map for sec %u\n", 3210 shdr_idx); 3211 return -LIBBPF_ERRNO__RELOC; 3212 } 3213 3214 reloc_desc->type = RELO_DATA; 3215 reloc_desc->insn_idx = insn_idx; 3216 reloc_desc->map_idx = map_idx; 3217 reloc_desc->sym_off = sym->st_value; 3218 return 0; 3219 } 3220 3221 static int 3222 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 3223 Elf_Data *data, struct bpf_object *obj) 3224 { 3225 Elf_Data *symbols = obj->efile.symbols; 3226 int err, i, nrels; 3227 3228 pr_debug("collecting relocating info for: '%s'\n", prog->section_name); 3229 nrels = shdr->sh_size / shdr->sh_entsize; 3230 3231 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 3232 if (!prog->reloc_desc) { 3233 pr_warn("failed to alloc memory in relocation\n"); 3234 return -ENOMEM; 3235 } 3236 prog->nr_reloc = nrels; 3237 3238 for (i = 0; i < nrels; i++) { 3239 const char *name; 3240 __u32 insn_idx; 3241 GElf_Sym sym; 3242 GElf_Rel rel; 3243 3244 if (!gelf_getrel(data, i, &rel)) { 3245 pr_warn("relocation: failed to get %d reloc\n", i); 3246 return -LIBBPF_ERRNO__FORMAT; 3247 } 3248 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 3249 pr_warn("relocation: symbol %"PRIx64" not found\n", 3250 GELF_R_SYM(rel.r_info)); 3251 return -LIBBPF_ERRNO__FORMAT; 3252 } 3253 if (rel.r_offset % sizeof(struct bpf_insn)) 3254 return -LIBBPF_ERRNO__FORMAT; 3255 3256 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 3257 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 3258 sym.st_name) ? : "<?>"; 3259 3260 pr_debug("relo for shdr %u, symb %zu, value %zu, type %d, bind %d, name %d (\'%s\'), insn %u\n", 3261 (__u32)sym.st_shndx, (size_t)GELF_R_SYM(rel.r_info), 3262 (size_t)sym.st_value, GELF_ST_TYPE(sym.st_info), 3263 GELF_ST_BIND(sym.st_info), sym.st_name, name, 3264 insn_idx); 3265 3266 err = bpf_program__record_reloc(prog, &prog->reloc_desc[i], 3267 insn_idx, name, &sym, &rel); 3268 if (err) 3269 return err; 3270 } 3271 return 0; 3272 } 3273 3274 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) 3275 { 3276 struct bpf_map_def *def = &map->def; 3277 __u32 key_type_id = 0, value_type_id = 0; 3278 int ret; 3279 3280 /* if it's BTF-defined map, we don't need to search for type IDs. 3281 * For struct_ops map, it does not need btf_key_type_id and 3282 * btf_value_type_id. 3283 */ 3284 if (map->sec_idx == obj->efile.btf_maps_shndx || 3285 bpf_map__is_struct_ops(map)) 3286 return 0; 3287 3288 if (!bpf_map__is_internal(map)) { 3289 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, 3290 def->value_size, &key_type_id, 3291 &value_type_id); 3292 } else { 3293 /* 3294 * LLVM annotates global data differently in BTF, that is, 3295 * only as '.data', '.bss' or '.rodata'. 3296 */ 3297 ret = btf__find_by_name(obj->btf, 3298 libbpf_type_to_btf_name[map->libbpf_type]); 3299 } 3300 if (ret < 0) 3301 return ret; 3302 3303 map->btf_key_type_id = key_type_id; 3304 map->btf_value_type_id = bpf_map__is_internal(map) ? 3305 ret : value_type_id; 3306 return 0; 3307 } 3308 3309 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 3310 { 3311 struct bpf_map_info info = {}; 3312 __u32 len = sizeof(info); 3313 int new_fd, err; 3314 char *new_name; 3315 3316 err = bpf_obj_get_info_by_fd(fd, &info, &len); 3317 if (err) 3318 return err; 3319 3320 new_name = strdup(info.name); 3321 if (!new_name) 3322 return -errno; 3323 3324 new_fd = open("/", O_RDONLY | O_CLOEXEC); 3325 if (new_fd < 0) { 3326 err = -errno; 3327 goto err_free_new_name; 3328 } 3329 3330 new_fd = dup3(fd, new_fd, O_CLOEXEC); 3331 if (new_fd < 0) { 3332 err = -errno; 3333 goto err_close_new_fd; 3334 } 3335 3336 err = zclose(map->fd); 3337 if (err) { 3338 err = -errno; 3339 goto err_close_new_fd; 3340 } 3341 free(map->name); 3342 3343 map->fd = new_fd; 3344 map->name = new_name; 3345 map->def.type = info.type; 3346 map->def.key_size = info.key_size; 3347 map->def.value_size = info.value_size; 3348 map->def.max_entries = info.max_entries; 3349 map->def.map_flags = info.map_flags; 3350 map->btf_key_type_id = info.btf_key_type_id; 3351 map->btf_value_type_id = info.btf_value_type_id; 3352 map->reused = true; 3353 3354 return 0; 3355 3356 err_close_new_fd: 3357 close(new_fd); 3358 err_free_new_name: 3359 free(new_name); 3360 return err; 3361 } 3362 3363 __u32 bpf_map__max_entries(const struct bpf_map *map) 3364 { 3365 return map->def.max_entries; 3366 } 3367 3368 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) 3369 { 3370 if (map->fd >= 0) 3371 return -EBUSY; 3372 map->def.max_entries = max_entries; 3373 return 0; 3374 } 3375 3376 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 3377 { 3378 if (!map || !max_entries) 3379 return -EINVAL; 3380 3381 return bpf_map__set_max_entries(map, max_entries); 3382 } 3383 3384 static int 3385 bpf_object__probe_loading(struct bpf_object *obj) 3386 { 3387 struct bpf_load_program_attr attr; 3388 char *cp, errmsg[STRERR_BUFSIZE]; 3389 struct bpf_insn insns[] = { 3390 BPF_MOV64_IMM(BPF_REG_0, 0), 3391 BPF_EXIT_INSN(), 3392 }; 3393 int ret; 3394 3395 /* make sure basic loading works */ 3396 3397 memset(&attr, 0, sizeof(attr)); 3398 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 3399 attr.insns = insns; 3400 attr.insns_cnt = ARRAY_SIZE(insns); 3401 attr.license = "GPL"; 3402 3403 ret = bpf_load_program_xattr(&attr, NULL, 0); 3404 if (ret < 0) { 3405 ret = errno; 3406 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 3407 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF " 3408 "program. Make sure your kernel supports BPF " 3409 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is " 3410 "set to big enough value.\n", __func__, cp, ret); 3411 return -ret; 3412 } 3413 close(ret); 3414 3415 return 0; 3416 } 3417 3418 static int 3419 bpf_object__probe_name(struct bpf_object *obj) 3420 { 3421 struct bpf_load_program_attr attr; 3422 struct bpf_insn insns[] = { 3423 BPF_MOV64_IMM(BPF_REG_0, 0), 3424 BPF_EXIT_INSN(), 3425 }; 3426 int ret; 3427 3428 /* make sure loading with name works */ 3429 3430 memset(&attr, 0, sizeof(attr)); 3431 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 3432 attr.insns = insns; 3433 attr.insns_cnt = ARRAY_SIZE(insns); 3434 attr.license = "GPL"; 3435 attr.name = "test"; 3436 ret = bpf_load_program_xattr(&attr, NULL, 0); 3437 if (ret >= 0) { 3438 obj->caps.name = 1; 3439 close(ret); 3440 } 3441 3442 return 0; 3443 } 3444 3445 static int 3446 bpf_object__probe_global_data(struct bpf_object *obj) 3447 { 3448 struct bpf_load_program_attr prg_attr; 3449 struct bpf_create_map_attr map_attr; 3450 char *cp, errmsg[STRERR_BUFSIZE]; 3451 struct bpf_insn insns[] = { 3452 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 3453 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 3454 BPF_MOV64_IMM(BPF_REG_0, 0), 3455 BPF_EXIT_INSN(), 3456 }; 3457 int ret, map; 3458 3459 memset(&map_attr, 0, sizeof(map_attr)); 3460 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 3461 map_attr.key_size = sizeof(int); 3462 map_attr.value_size = 32; 3463 map_attr.max_entries = 1; 3464 3465 map = bpf_create_map_xattr(&map_attr); 3466 if (map < 0) { 3467 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 3468 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 3469 __func__, cp, errno); 3470 return -errno; 3471 } 3472 3473 insns[0].imm = map; 3474 3475 memset(&prg_attr, 0, sizeof(prg_attr)); 3476 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 3477 prg_attr.insns = insns; 3478 prg_attr.insns_cnt = ARRAY_SIZE(insns); 3479 prg_attr.license = "GPL"; 3480 3481 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 3482 if (ret >= 0) { 3483 obj->caps.global_data = 1; 3484 close(ret); 3485 } 3486 3487 close(map); 3488 return 0; 3489 } 3490 3491 static int bpf_object__probe_btf_func(struct bpf_object *obj) 3492 { 3493 static const char strs[] = "\0int\0x\0a"; 3494 /* void x(int a) {} */ 3495 __u32 types[] = { 3496 /* int */ 3497 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 3498 /* FUNC_PROTO */ /* [2] */ 3499 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 3500 BTF_PARAM_ENC(7, 1), 3501 /* FUNC x */ /* [3] */ 3502 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 3503 }; 3504 int btf_fd; 3505 3506 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 3507 strs, sizeof(strs)); 3508 if (btf_fd >= 0) { 3509 obj->caps.btf_func = 1; 3510 close(btf_fd); 3511 return 1; 3512 } 3513 3514 return 0; 3515 } 3516 3517 static int bpf_object__probe_btf_func_global(struct bpf_object *obj) 3518 { 3519 static const char strs[] = "\0int\0x\0a"; 3520 /* static void x(int a) {} */ 3521 __u32 types[] = { 3522 /* int */ 3523 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 3524 /* FUNC_PROTO */ /* [2] */ 3525 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 3526 BTF_PARAM_ENC(7, 1), 3527 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ 3528 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), 3529 }; 3530 int btf_fd; 3531 3532 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 3533 strs, sizeof(strs)); 3534 if (btf_fd >= 0) { 3535 obj->caps.btf_func_global = 1; 3536 close(btf_fd); 3537 return 1; 3538 } 3539 3540 return 0; 3541 } 3542 3543 static int bpf_object__probe_btf_datasec(struct bpf_object *obj) 3544 { 3545 static const char strs[] = "\0x\0.data"; 3546 /* static int a; */ 3547 __u32 types[] = { 3548 /* int */ 3549 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 3550 /* VAR x */ /* [2] */ 3551 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 3552 BTF_VAR_STATIC, 3553 /* DATASEC val */ /* [3] */ 3554 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 3555 BTF_VAR_SECINFO_ENC(2, 0, 4), 3556 }; 3557 int btf_fd; 3558 3559 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 3560 strs, sizeof(strs)); 3561 if (btf_fd >= 0) { 3562 obj->caps.btf_datasec = 1; 3563 close(btf_fd); 3564 return 1; 3565 } 3566 3567 return 0; 3568 } 3569 3570 static int bpf_object__probe_array_mmap(struct bpf_object *obj) 3571 { 3572 struct bpf_create_map_attr attr = { 3573 .map_type = BPF_MAP_TYPE_ARRAY, 3574 .map_flags = BPF_F_MMAPABLE, 3575 .key_size = sizeof(int), 3576 .value_size = sizeof(int), 3577 .max_entries = 1, 3578 }; 3579 int fd; 3580 3581 fd = bpf_create_map_xattr(&attr); 3582 if (fd >= 0) { 3583 obj->caps.array_mmap = 1; 3584 close(fd); 3585 return 1; 3586 } 3587 3588 return 0; 3589 } 3590 3591 static int 3592 bpf_object__probe_exp_attach_type(struct bpf_object *obj) 3593 { 3594 struct bpf_load_program_attr attr; 3595 struct bpf_insn insns[] = { 3596 BPF_MOV64_IMM(BPF_REG_0, 0), 3597 BPF_EXIT_INSN(), 3598 }; 3599 int fd; 3600 3601 memset(&attr, 0, sizeof(attr)); 3602 /* use any valid combination of program type and (optional) 3603 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) 3604 * to see if kernel supports expected_attach_type field for 3605 * BPF_PROG_LOAD command 3606 */ 3607 attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK; 3608 attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE; 3609 attr.insns = insns; 3610 attr.insns_cnt = ARRAY_SIZE(insns); 3611 attr.license = "GPL"; 3612 3613 fd = bpf_load_program_xattr(&attr, NULL, 0); 3614 if (fd >= 0) { 3615 obj->caps.exp_attach_type = 1; 3616 close(fd); 3617 return 1; 3618 } 3619 return 0; 3620 } 3621 3622 static int 3623 bpf_object__probe_caps(struct bpf_object *obj) 3624 { 3625 int (*probe_fn[])(struct bpf_object *obj) = { 3626 bpf_object__probe_name, 3627 bpf_object__probe_global_data, 3628 bpf_object__probe_btf_func, 3629 bpf_object__probe_btf_func_global, 3630 bpf_object__probe_btf_datasec, 3631 bpf_object__probe_array_mmap, 3632 bpf_object__probe_exp_attach_type, 3633 }; 3634 int i, ret; 3635 3636 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { 3637 ret = probe_fn[i](obj); 3638 if (ret < 0) 3639 pr_debug("Probe #%d failed with %d.\n", i, ret); 3640 } 3641 3642 return 0; 3643 } 3644 3645 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) 3646 { 3647 struct bpf_map_info map_info = {}; 3648 char msg[STRERR_BUFSIZE]; 3649 __u32 map_info_len; 3650 3651 map_info_len = sizeof(map_info); 3652 3653 if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) { 3654 pr_warn("failed to get map info for map FD %d: %s\n", 3655 map_fd, libbpf_strerror_r(errno, msg, sizeof(msg))); 3656 return false; 3657 } 3658 3659 return (map_info.type == map->def.type && 3660 map_info.key_size == map->def.key_size && 3661 map_info.value_size == map->def.value_size && 3662 map_info.max_entries == map->def.max_entries && 3663 map_info.map_flags == map->def.map_flags); 3664 } 3665 3666 static int 3667 bpf_object__reuse_map(struct bpf_map *map) 3668 { 3669 char *cp, errmsg[STRERR_BUFSIZE]; 3670 int err, pin_fd; 3671 3672 pin_fd = bpf_obj_get(map->pin_path); 3673 if (pin_fd < 0) { 3674 err = -errno; 3675 if (err == -ENOENT) { 3676 pr_debug("found no pinned map to reuse at '%s'\n", 3677 map->pin_path); 3678 return 0; 3679 } 3680 3681 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 3682 pr_warn("couldn't retrieve pinned map '%s': %s\n", 3683 map->pin_path, cp); 3684 return err; 3685 } 3686 3687 if (!map_is_reuse_compat(map, pin_fd)) { 3688 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", 3689 map->pin_path); 3690 close(pin_fd); 3691 return -EINVAL; 3692 } 3693 3694 err = bpf_map__reuse_fd(map, pin_fd); 3695 if (err) { 3696 close(pin_fd); 3697 return err; 3698 } 3699 map->pinned = true; 3700 pr_debug("reused pinned map at '%s'\n", map->pin_path); 3701 3702 return 0; 3703 } 3704 3705 static int 3706 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 3707 { 3708 enum libbpf_map_type map_type = map->libbpf_type; 3709 char *cp, errmsg[STRERR_BUFSIZE]; 3710 int err, zero = 0; 3711 3712 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); 3713 if (err) { 3714 err = -errno; 3715 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 3716 pr_warn("Error setting initial map(%s) contents: %s\n", 3717 map->name, cp); 3718 return err; 3719 } 3720 3721 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ 3722 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { 3723 err = bpf_map_freeze(map->fd); 3724 if (err) { 3725 err = -errno; 3726 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 3727 pr_warn("Error freezing map(%s) as read-only: %s\n", 3728 map->name, cp); 3729 return err; 3730 } 3731 } 3732 return 0; 3733 } 3734 3735 static void bpf_map__destroy(struct bpf_map *map); 3736 3737 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map) 3738 { 3739 struct bpf_create_map_attr create_attr; 3740 struct bpf_map_def *def = &map->def; 3741 3742 memset(&create_attr, 0, sizeof(create_attr)); 3743 3744 if (obj->caps.name) 3745 create_attr.name = map->name; 3746 create_attr.map_ifindex = map->map_ifindex; 3747 create_attr.map_type = def->type; 3748 create_attr.map_flags = def->map_flags; 3749 create_attr.key_size = def->key_size; 3750 create_attr.value_size = def->value_size; 3751 create_attr.numa_node = map->numa_node; 3752 3753 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) { 3754 int nr_cpus; 3755 3756 nr_cpus = libbpf_num_possible_cpus(); 3757 if (nr_cpus < 0) { 3758 pr_warn("map '%s': failed to determine number of system CPUs: %d\n", 3759 map->name, nr_cpus); 3760 return nr_cpus; 3761 } 3762 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); 3763 create_attr.max_entries = nr_cpus; 3764 } else { 3765 create_attr.max_entries = def->max_entries; 3766 } 3767 3768 if (bpf_map__is_struct_ops(map)) 3769 create_attr.btf_vmlinux_value_type_id = 3770 map->btf_vmlinux_value_type_id; 3771 3772 create_attr.btf_fd = 0; 3773 create_attr.btf_key_type_id = 0; 3774 create_attr.btf_value_type_id = 0; 3775 if (obj->btf && !bpf_map_find_btf_info(obj, map)) { 3776 create_attr.btf_fd = btf__fd(obj->btf); 3777 create_attr.btf_key_type_id = map->btf_key_type_id; 3778 create_attr.btf_value_type_id = map->btf_value_type_id; 3779 } 3780 3781 if (bpf_map_type__is_map_in_map(def->type)) { 3782 if (map->inner_map) { 3783 int err; 3784 3785 err = bpf_object__create_map(obj, map->inner_map); 3786 if (err) { 3787 pr_warn("map '%s': failed to create inner map: %d\n", 3788 map->name, err); 3789 return err; 3790 } 3791 map->inner_map_fd = bpf_map__fd(map->inner_map); 3792 } 3793 if (map->inner_map_fd >= 0) 3794 create_attr.inner_map_fd = map->inner_map_fd; 3795 } 3796 3797 map->fd = bpf_create_map_xattr(&create_attr); 3798 if (map->fd < 0 && (create_attr.btf_key_type_id || 3799 create_attr.btf_value_type_id)) { 3800 char *cp, errmsg[STRERR_BUFSIZE]; 3801 int err = -errno; 3802 3803 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 3804 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 3805 map->name, cp, err); 3806 create_attr.btf_fd = 0; 3807 create_attr.btf_key_type_id = 0; 3808 create_attr.btf_value_type_id = 0; 3809 map->btf_key_type_id = 0; 3810 map->btf_value_type_id = 0; 3811 map->fd = bpf_create_map_xattr(&create_attr); 3812 } 3813 3814 if (map->fd < 0) 3815 return -errno; 3816 3817 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { 3818 bpf_map__destroy(map->inner_map); 3819 zfree(&map->inner_map); 3820 } 3821 3822 return 0; 3823 } 3824 3825 static int 3826 bpf_object__create_maps(struct bpf_object *obj) 3827 { 3828 struct bpf_map *map; 3829 char *cp, errmsg[STRERR_BUFSIZE]; 3830 unsigned int i, j; 3831 int err; 3832 3833 for (i = 0; i < obj->nr_maps; i++) { 3834 map = &obj->maps[i]; 3835 3836 if (map->pin_path) { 3837 err = bpf_object__reuse_map(map); 3838 if (err) { 3839 pr_warn("map '%s': error reusing pinned map\n", 3840 map->name); 3841 goto err_out; 3842 } 3843 } 3844 3845 if (map->fd >= 0) { 3846 pr_debug("map '%s': skipping creation (preset fd=%d)\n", 3847 map->name, map->fd); 3848 continue; 3849 } 3850 3851 err = bpf_object__create_map(obj, map); 3852 if (err) 3853 goto err_out; 3854 3855 pr_debug("map '%s': created successfully, fd=%d\n", map->name, 3856 map->fd); 3857 3858 if (bpf_map__is_internal(map)) { 3859 err = bpf_object__populate_internal_map(obj, map); 3860 if (err < 0) { 3861 zclose(map->fd); 3862 goto err_out; 3863 } 3864 } 3865 3866 if (map->init_slots_sz) { 3867 for (j = 0; j < map->init_slots_sz; j++) { 3868 const struct bpf_map *targ_map; 3869 int fd; 3870 3871 if (!map->init_slots[j]) 3872 continue; 3873 3874 targ_map = map->init_slots[j]; 3875 fd = bpf_map__fd(targ_map); 3876 err = bpf_map_update_elem(map->fd, &j, &fd, 0); 3877 if (err) { 3878 err = -errno; 3879 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n", 3880 map->name, j, targ_map->name, 3881 fd, err); 3882 goto err_out; 3883 } 3884 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", 3885 map->name, j, targ_map->name, fd); 3886 } 3887 zfree(&map->init_slots); 3888 map->init_slots_sz = 0; 3889 } 3890 3891 if (map->pin_path && !map->pinned) { 3892 err = bpf_map__pin(map, NULL); 3893 if (err) { 3894 pr_warn("map '%s': failed to auto-pin at '%s': %d\n", 3895 map->name, map->pin_path, err); 3896 zclose(map->fd); 3897 goto err_out; 3898 } 3899 } 3900 } 3901 3902 return 0; 3903 3904 err_out: 3905 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 3906 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err); 3907 pr_perm_msg(err); 3908 for (j = 0; j < i; j++) 3909 zclose(obj->maps[j].fd); 3910 return err; 3911 } 3912 3913 static int 3914 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 3915 void *btf_prog_info, const char *info_name) 3916 { 3917 if (err != -ENOENT) { 3918 pr_warn("Error in loading %s for sec %s.\n", 3919 info_name, prog->section_name); 3920 return err; 3921 } 3922 3923 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 3924 3925 if (btf_prog_info) { 3926 /* 3927 * Some info has already been found but has problem 3928 * in the last btf_ext reloc. Must have to error out. 3929 */ 3930 pr_warn("Error in relocating %s for sec %s.\n", 3931 info_name, prog->section_name); 3932 return err; 3933 } 3934 3935 /* Have problem loading the very first info. Ignore the rest. */ 3936 pr_warn("Cannot find %s for main program sec %s. Ignore all %s.\n", 3937 info_name, prog->section_name, info_name); 3938 return 0; 3939 } 3940 3941 static int 3942 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 3943 const char *section_name, __u32 insn_offset) 3944 { 3945 int err; 3946 3947 if (!insn_offset || prog->func_info) { 3948 /* 3949 * !insn_offset => main program 3950 * 3951 * For sub prog, the main program's func_info has to 3952 * be loaded first (i.e. prog->func_info != NULL) 3953 */ 3954 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 3955 section_name, insn_offset, 3956 &prog->func_info, 3957 &prog->func_info_cnt); 3958 if (err) 3959 return check_btf_ext_reloc_err(prog, err, 3960 prog->func_info, 3961 "bpf_func_info"); 3962 3963 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 3964 } 3965 3966 if (!insn_offset || prog->line_info) { 3967 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 3968 section_name, insn_offset, 3969 &prog->line_info, 3970 &prog->line_info_cnt); 3971 if (err) 3972 return check_btf_ext_reloc_err(prog, err, 3973 prog->line_info, 3974 "bpf_line_info"); 3975 3976 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 3977 } 3978 3979 return 0; 3980 } 3981 3982 #define BPF_CORE_SPEC_MAX_LEN 64 3983 3984 /* represents BPF CO-RE field or array element accessor */ 3985 struct bpf_core_accessor { 3986 __u32 type_id; /* struct/union type or array element type */ 3987 __u32 idx; /* field index or array index */ 3988 const char *name; /* field name or NULL for array accessor */ 3989 }; 3990 3991 struct bpf_core_spec { 3992 const struct btf *btf; 3993 /* high-level spec: named fields and array indices only */ 3994 struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN]; 3995 /* high-level spec length */ 3996 int len; 3997 /* raw, low-level spec: 1-to-1 with accessor spec string */ 3998 int raw_spec[BPF_CORE_SPEC_MAX_LEN]; 3999 /* raw spec length */ 4000 int raw_len; 4001 /* field bit offset represented by spec */ 4002 __u32 bit_offset; 4003 }; 4004 4005 static bool str_is_empty(const char *s) 4006 { 4007 return !s || !s[0]; 4008 } 4009 4010 static bool is_flex_arr(const struct btf *btf, 4011 const struct bpf_core_accessor *acc, 4012 const struct btf_array *arr) 4013 { 4014 const struct btf_type *t; 4015 4016 /* not a flexible array, if not inside a struct or has non-zero size */ 4017 if (!acc->name || arr->nelems > 0) 4018 return false; 4019 4020 /* has to be the last member of enclosing struct */ 4021 t = btf__type_by_id(btf, acc->type_id); 4022 return acc->idx == btf_vlen(t) - 1; 4023 } 4024 4025 /* 4026 * Turn bpf_field_reloc into a low- and high-level spec representation, 4027 * validating correctness along the way, as well as calculating resulting 4028 * field bit offset, specified by accessor string. Low-level spec captures 4029 * every single level of nestedness, including traversing anonymous 4030 * struct/union members. High-level one only captures semantically meaningful 4031 * "turning points": named fields and array indicies. 4032 * E.g., for this case: 4033 * 4034 * struct sample { 4035 * int __unimportant; 4036 * struct { 4037 * int __1; 4038 * int __2; 4039 * int a[7]; 4040 * }; 4041 * }; 4042 * 4043 * struct sample *s = ...; 4044 * 4045 * int x = &s->a[3]; // access string = '0:1:2:3' 4046 * 4047 * Low-level spec has 1:1 mapping with each element of access string (it's 4048 * just a parsed access string representation): [0, 1, 2, 3]. 4049 * 4050 * High-level spec will capture only 3 points: 4051 * - intial zero-index access by pointer (&s->... is the same as &s[0]...); 4052 * - field 'a' access (corresponds to '2' in low-level spec); 4053 * - array element #3 access (corresponds to '3' in low-level spec). 4054 * 4055 */ 4056 static int bpf_core_spec_parse(const struct btf *btf, 4057 __u32 type_id, 4058 const char *spec_str, 4059 struct bpf_core_spec *spec) 4060 { 4061 int access_idx, parsed_len, i; 4062 struct bpf_core_accessor *acc; 4063 const struct btf_type *t; 4064 const char *name; 4065 __u32 id; 4066 __s64 sz; 4067 4068 if (str_is_empty(spec_str) || *spec_str == ':') 4069 return -EINVAL; 4070 4071 memset(spec, 0, sizeof(*spec)); 4072 spec->btf = btf; 4073 4074 /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */ 4075 while (*spec_str) { 4076 if (*spec_str == ':') 4077 ++spec_str; 4078 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1) 4079 return -EINVAL; 4080 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 4081 return -E2BIG; 4082 spec_str += parsed_len; 4083 spec->raw_spec[spec->raw_len++] = access_idx; 4084 } 4085 4086 if (spec->raw_len == 0) 4087 return -EINVAL; 4088 4089 /* first spec value is always reloc type array index */ 4090 t = skip_mods_and_typedefs(btf, type_id, &id); 4091 if (!t) 4092 return -EINVAL; 4093 4094 access_idx = spec->raw_spec[0]; 4095 spec->spec[0].type_id = id; 4096 spec->spec[0].idx = access_idx; 4097 spec->len++; 4098 4099 sz = btf__resolve_size(btf, id); 4100 if (sz < 0) 4101 return sz; 4102 spec->bit_offset = access_idx * sz * 8; 4103 4104 for (i = 1; i < spec->raw_len; i++) { 4105 t = skip_mods_and_typedefs(btf, id, &id); 4106 if (!t) 4107 return -EINVAL; 4108 4109 access_idx = spec->raw_spec[i]; 4110 acc = &spec->spec[spec->len]; 4111 4112 if (btf_is_composite(t)) { 4113 const struct btf_member *m; 4114 __u32 bit_offset; 4115 4116 if (access_idx >= btf_vlen(t)) 4117 return -EINVAL; 4118 4119 bit_offset = btf_member_bit_offset(t, access_idx); 4120 spec->bit_offset += bit_offset; 4121 4122 m = btf_members(t) + access_idx; 4123 if (m->name_off) { 4124 name = btf__name_by_offset(btf, m->name_off); 4125 if (str_is_empty(name)) 4126 return -EINVAL; 4127 4128 acc->type_id = id; 4129 acc->idx = access_idx; 4130 acc->name = name; 4131 spec->len++; 4132 } 4133 4134 id = m->type; 4135 } else if (btf_is_array(t)) { 4136 const struct btf_array *a = btf_array(t); 4137 bool flex; 4138 4139 t = skip_mods_and_typedefs(btf, a->type, &id); 4140 if (!t) 4141 return -EINVAL; 4142 4143 flex = is_flex_arr(btf, acc - 1, a); 4144 if (!flex && access_idx >= a->nelems) 4145 return -EINVAL; 4146 4147 spec->spec[spec->len].type_id = id; 4148 spec->spec[spec->len].idx = access_idx; 4149 spec->len++; 4150 4151 sz = btf__resolve_size(btf, id); 4152 if (sz < 0) 4153 return sz; 4154 spec->bit_offset += access_idx * sz * 8; 4155 } else { 4156 pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n", 4157 type_id, spec_str, i, id, btf_kind(t)); 4158 return -EINVAL; 4159 } 4160 } 4161 4162 return 0; 4163 } 4164 4165 static bool bpf_core_is_flavor_sep(const char *s) 4166 { 4167 /* check X___Y name pattern, where X and Y are not underscores */ 4168 return s[0] != '_' && /* X */ 4169 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 4170 s[4] != '_'; /* Y */ 4171 } 4172 4173 /* Given 'some_struct_name___with_flavor' return the length of a name prefix 4174 * before last triple underscore. Struct name part after last triple 4175 * underscore is ignored by BPF CO-RE relocation during relocation matching. 4176 */ 4177 static size_t bpf_core_essential_name_len(const char *name) 4178 { 4179 size_t n = strlen(name); 4180 int i; 4181 4182 for (i = n - 5; i >= 0; i--) { 4183 if (bpf_core_is_flavor_sep(name + i)) 4184 return i + 1; 4185 } 4186 return n; 4187 } 4188 4189 /* dynamically sized list of type IDs */ 4190 struct ids_vec { 4191 __u32 *data; 4192 int len; 4193 }; 4194 4195 static void bpf_core_free_cands(struct ids_vec *cand_ids) 4196 { 4197 free(cand_ids->data); 4198 free(cand_ids); 4199 } 4200 4201 static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf, 4202 __u32 local_type_id, 4203 const struct btf *targ_btf) 4204 { 4205 size_t local_essent_len, targ_essent_len; 4206 const char *local_name, *targ_name; 4207 const struct btf_type *t; 4208 struct ids_vec *cand_ids; 4209 __u32 *new_ids; 4210 int i, err, n; 4211 4212 t = btf__type_by_id(local_btf, local_type_id); 4213 if (!t) 4214 return ERR_PTR(-EINVAL); 4215 4216 local_name = btf__name_by_offset(local_btf, t->name_off); 4217 if (str_is_empty(local_name)) 4218 return ERR_PTR(-EINVAL); 4219 local_essent_len = bpf_core_essential_name_len(local_name); 4220 4221 cand_ids = calloc(1, sizeof(*cand_ids)); 4222 if (!cand_ids) 4223 return ERR_PTR(-ENOMEM); 4224 4225 n = btf__get_nr_types(targ_btf); 4226 for (i = 1; i <= n; i++) { 4227 t = btf__type_by_id(targ_btf, i); 4228 targ_name = btf__name_by_offset(targ_btf, t->name_off); 4229 if (str_is_empty(targ_name)) 4230 continue; 4231 4232 t = skip_mods_and_typedefs(targ_btf, i, NULL); 4233 if (!btf_is_composite(t) && !btf_is_array(t)) 4234 continue; 4235 4236 targ_essent_len = bpf_core_essential_name_len(targ_name); 4237 if (targ_essent_len != local_essent_len) 4238 continue; 4239 4240 if (strncmp(local_name, targ_name, local_essent_len) == 0) { 4241 pr_debug("[%d] %s: found candidate [%d] %s\n", 4242 local_type_id, local_name, i, targ_name); 4243 new_ids = reallocarray(cand_ids->data, 4244 cand_ids->len + 1, 4245 sizeof(*cand_ids->data)); 4246 if (!new_ids) { 4247 err = -ENOMEM; 4248 goto err_out; 4249 } 4250 cand_ids->data = new_ids; 4251 cand_ids->data[cand_ids->len++] = i; 4252 } 4253 } 4254 return cand_ids; 4255 err_out: 4256 bpf_core_free_cands(cand_ids); 4257 return ERR_PTR(err); 4258 } 4259 4260 /* Check two types for compatibility, skipping const/volatile/restrict and 4261 * typedefs, to ensure we are relocating compatible entities: 4262 * - any two STRUCTs/UNIONs are compatible and can be mixed; 4263 * - any two FWDs are compatible, if their names match (modulo flavor suffix); 4264 * - any two PTRs are always compatible; 4265 * - for ENUMs, names should be the same (ignoring flavor suffix) or at 4266 * least one of enums should be anonymous; 4267 * - for ENUMs, check sizes, names are ignored; 4268 * - for INT, size and signedness are ignored; 4269 * - for ARRAY, dimensionality is ignored, element types are checked for 4270 * compatibility recursively; 4271 * - everything else shouldn't be ever a target of relocation. 4272 * These rules are not set in stone and probably will be adjusted as we get 4273 * more experience with using BPF CO-RE relocations. 4274 */ 4275 static int bpf_core_fields_are_compat(const struct btf *local_btf, 4276 __u32 local_id, 4277 const struct btf *targ_btf, 4278 __u32 targ_id) 4279 { 4280 const struct btf_type *local_type, *targ_type; 4281 4282 recur: 4283 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); 4284 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 4285 if (!local_type || !targ_type) 4286 return -EINVAL; 4287 4288 if (btf_is_composite(local_type) && btf_is_composite(targ_type)) 4289 return 1; 4290 if (btf_kind(local_type) != btf_kind(targ_type)) 4291 return 0; 4292 4293 switch (btf_kind(local_type)) { 4294 case BTF_KIND_PTR: 4295 return 1; 4296 case BTF_KIND_FWD: 4297 case BTF_KIND_ENUM: { 4298 const char *local_name, *targ_name; 4299 size_t local_len, targ_len; 4300 4301 local_name = btf__name_by_offset(local_btf, 4302 local_type->name_off); 4303 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off); 4304 local_len = bpf_core_essential_name_len(local_name); 4305 targ_len = bpf_core_essential_name_len(targ_name); 4306 /* one of them is anonymous or both w/ same flavor-less names */ 4307 return local_len == 0 || targ_len == 0 || 4308 (local_len == targ_len && 4309 strncmp(local_name, targ_name, local_len) == 0); 4310 } 4311 case BTF_KIND_INT: 4312 /* just reject deprecated bitfield-like integers; all other 4313 * integers are by default compatible between each other 4314 */ 4315 return btf_int_offset(local_type) == 0 && 4316 btf_int_offset(targ_type) == 0; 4317 case BTF_KIND_ARRAY: 4318 local_id = btf_array(local_type)->type; 4319 targ_id = btf_array(targ_type)->type; 4320 goto recur; 4321 default: 4322 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n", 4323 btf_kind(local_type), local_id, targ_id); 4324 return 0; 4325 } 4326 } 4327 4328 /* 4329 * Given single high-level named field accessor in local type, find 4330 * corresponding high-level accessor for a target type. Along the way, 4331 * maintain low-level spec for target as well. Also keep updating target 4332 * bit offset. 4333 * 4334 * Searching is performed through recursive exhaustive enumeration of all 4335 * fields of a struct/union. If there are any anonymous (embedded) 4336 * structs/unions, they are recursively searched as well. If field with 4337 * desired name is found, check compatibility between local and target types, 4338 * before returning result. 4339 * 4340 * 1 is returned, if field is found. 4341 * 0 is returned if no compatible field is found. 4342 * <0 is returned on error. 4343 */ 4344 static int bpf_core_match_member(const struct btf *local_btf, 4345 const struct bpf_core_accessor *local_acc, 4346 const struct btf *targ_btf, 4347 __u32 targ_id, 4348 struct bpf_core_spec *spec, 4349 __u32 *next_targ_id) 4350 { 4351 const struct btf_type *local_type, *targ_type; 4352 const struct btf_member *local_member, *m; 4353 const char *local_name, *targ_name; 4354 __u32 local_id; 4355 int i, n, found; 4356 4357 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); 4358 if (!targ_type) 4359 return -EINVAL; 4360 if (!btf_is_composite(targ_type)) 4361 return 0; 4362 4363 local_id = local_acc->type_id; 4364 local_type = btf__type_by_id(local_btf, local_id); 4365 local_member = btf_members(local_type) + local_acc->idx; 4366 local_name = btf__name_by_offset(local_btf, local_member->name_off); 4367 4368 n = btf_vlen(targ_type); 4369 m = btf_members(targ_type); 4370 for (i = 0; i < n; i++, m++) { 4371 __u32 bit_offset; 4372 4373 bit_offset = btf_member_bit_offset(targ_type, i); 4374 4375 /* too deep struct/union/array nesting */ 4376 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 4377 return -E2BIG; 4378 4379 /* speculate this member will be the good one */ 4380 spec->bit_offset += bit_offset; 4381 spec->raw_spec[spec->raw_len++] = i; 4382 4383 targ_name = btf__name_by_offset(targ_btf, m->name_off); 4384 if (str_is_empty(targ_name)) { 4385 /* embedded struct/union, we need to go deeper */ 4386 found = bpf_core_match_member(local_btf, local_acc, 4387 targ_btf, m->type, 4388 spec, next_targ_id); 4389 if (found) /* either found or error */ 4390 return found; 4391 } else if (strcmp(local_name, targ_name) == 0) { 4392 /* matching named field */ 4393 struct bpf_core_accessor *targ_acc; 4394 4395 targ_acc = &spec->spec[spec->len++]; 4396 targ_acc->type_id = targ_id; 4397 targ_acc->idx = i; 4398 targ_acc->name = targ_name; 4399 4400 *next_targ_id = m->type; 4401 found = bpf_core_fields_are_compat(local_btf, 4402 local_member->type, 4403 targ_btf, m->type); 4404 if (!found) 4405 spec->len--; /* pop accessor */ 4406 return found; 4407 } 4408 /* member turned out not to be what we looked for */ 4409 spec->bit_offset -= bit_offset; 4410 spec->raw_len--; 4411 } 4412 4413 return 0; 4414 } 4415 4416 /* 4417 * Try to match local spec to a target type and, if successful, produce full 4418 * target spec (high-level, low-level + bit offset). 4419 */ 4420 static int bpf_core_spec_match(struct bpf_core_spec *local_spec, 4421 const struct btf *targ_btf, __u32 targ_id, 4422 struct bpf_core_spec *targ_spec) 4423 { 4424 const struct btf_type *targ_type; 4425 const struct bpf_core_accessor *local_acc; 4426 struct bpf_core_accessor *targ_acc; 4427 int i, sz, matched; 4428 4429 memset(targ_spec, 0, sizeof(*targ_spec)); 4430 targ_spec->btf = targ_btf; 4431 4432 local_acc = &local_spec->spec[0]; 4433 targ_acc = &targ_spec->spec[0]; 4434 4435 for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) { 4436 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, 4437 &targ_id); 4438 if (!targ_type) 4439 return -EINVAL; 4440 4441 if (local_acc->name) { 4442 matched = bpf_core_match_member(local_spec->btf, 4443 local_acc, 4444 targ_btf, targ_id, 4445 targ_spec, &targ_id); 4446 if (matched <= 0) 4447 return matched; 4448 } else { 4449 /* for i=0, targ_id is already treated as array element 4450 * type (because it's the original struct), for others 4451 * we should find array element type first 4452 */ 4453 if (i > 0) { 4454 const struct btf_array *a; 4455 bool flex; 4456 4457 if (!btf_is_array(targ_type)) 4458 return 0; 4459 4460 a = btf_array(targ_type); 4461 flex = is_flex_arr(targ_btf, targ_acc - 1, a); 4462 if (!flex && local_acc->idx >= a->nelems) 4463 return 0; 4464 if (!skip_mods_and_typedefs(targ_btf, a->type, 4465 &targ_id)) 4466 return -EINVAL; 4467 } 4468 4469 /* too deep struct/union/array nesting */ 4470 if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN) 4471 return -E2BIG; 4472 4473 targ_acc->type_id = targ_id; 4474 targ_acc->idx = local_acc->idx; 4475 targ_acc->name = NULL; 4476 targ_spec->len++; 4477 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx; 4478 targ_spec->raw_len++; 4479 4480 sz = btf__resolve_size(targ_btf, targ_id); 4481 if (sz < 0) 4482 return sz; 4483 targ_spec->bit_offset += local_acc->idx * sz * 8; 4484 } 4485 } 4486 4487 return 1; 4488 } 4489 4490 static int bpf_core_calc_field_relo(const struct bpf_program *prog, 4491 const struct bpf_field_reloc *relo, 4492 const struct bpf_core_spec *spec, 4493 __u32 *val, bool *validate) 4494 { 4495 const struct bpf_core_accessor *acc = &spec->spec[spec->len - 1]; 4496 const struct btf_type *t = btf__type_by_id(spec->btf, acc->type_id); 4497 __u32 byte_off, byte_sz, bit_off, bit_sz; 4498 const struct btf_member *m; 4499 const struct btf_type *mt; 4500 bool bitfield; 4501 __s64 sz; 4502 4503 /* a[n] accessor needs special handling */ 4504 if (!acc->name) { 4505 if (relo->kind == BPF_FIELD_BYTE_OFFSET) { 4506 *val = spec->bit_offset / 8; 4507 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) { 4508 sz = btf__resolve_size(spec->btf, acc->type_id); 4509 if (sz < 0) 4510 return -EINVAL; 4511 *val = sz; 4512 } else { 4513 pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n", 4514 bpf_program__title(prog, false), 4515 relo->kind, relo->insn_off / 8); 4516 return -EINVAL; 4517 } 4518 if (validate) 4519 *validate = true; 4520 return 0; 4521 } 4522 4523 m = btf_members(t) + acc->idx; 4524 mt = skip_mods_and_typedefs(spec->btf, m->type, NULL); 4525 bit_off = spec->bit_offset; 4526 bit_sz = btf_member_bitfield_size(t, acc->idx); 4527 4528 bitfield = bit_sz > 0; 4529 if (bitfield) { 4530 byte_sz = mt->size; 4531 byte_off = bit_off / 8 / byte_sz * byte_sz; 4532 /* figure out smallest int size necessary for bitfield load */ 4533 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) { 4534 if (byte_sz >= 8) { 4535 /* bitfield can't be read with 64-bit read */ 4536 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n", 4537 bpf_program__title(prog, false), 4538 relo->kind, relo->insn_off / 8); 4539 return -E2BIG; 4540 } 4541 byte_sz *= 2; 4542 byte_off = bit_off / 8 / byte_sz * byte_sz; 4543 } 4544 } else { 4545 sz = btf__resolve_size(spec->btf, m->type); 4546 if (sz < 0) 4547 return -EINVAL; 4548 byte_sz = sz; 4549 byte_off = spec->bit_offset / 8; 4550 bit_sz = byte_sz * 8; 4551 } 4552 4553 /* for bitfields, all the relocatable aspects are ambiguous and we 4554 * might disagree with compiler, so turn off validation of expected 4555 * value, except for signedness 4556 */ 4557 if (validate) 4558 *validate = !bitfield; 4559 4560 switch (relo->kind) { 4561 case BPF_FIELD_BYTE_OFFSET: 4562 *val = byte_off; 4563 break; 4564 case BPF_FIELD_BYTE_SIZE: 4565 *val = byte_sz; 4566 break; 4567 case BPF_FIELD_SIGNED: 4568 /* enums will be assumed unsigned */ 4569 *val = btf_is_enum(mt) || 4570 (btf_int_encoding(mt) & BTF_INT_SIGNED); 4571 if (validate) 4572 *validate = true; /* signedness is never ambiguous */ 4573 break; 4574 case BPF_FIELD_LSHIFT_U64: 4575 #if __BYTE_ORDER == __LITTLE_ENDIAN 4576 *val = 64 - (bit_off + bit_sz - byte_off * 8); 4577 #else 4578 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8); 4579 #endif 4580 break; 4581 case BPF_FIELD_RSHIFT_U64: 4582 *val = 64 - bit_sz; 4583 if (validate) 4584 *validate = true; /* right shift is never ambiguous */ 4585 break; 4586 case BPF_FIELD_EXISTS: 4587 default: 4588 pr_warn("prog '%s': unknown relo %d at insn #%d\n", 4589 bpf_program__title(prog, false), 4590 relo->kind, relo->insn_off / 8); 4591 return -EINVAL; 4592 } 4593 4594 return 0; 4595 } 4596 4597 /* 4598 * Patch relocatable BPF instruction. 4599 * 4600 * Patched value is determined by relocation kind and target specification. 4601 * For field existence relocation target spec will be NULL if field is not 4602 * found. 4603 * Expected insn->imm value is determined using relocation kind and local 4604 * spec, and is checked before patching instruction. If actual insn->imm value 4605 * is wrong, bail out with error. 4606 * 4607 * Currently three kinds of BPF instructions are supported: 4608 * 1. rX = <imm> (assignment with immediate operand); 4609 * 2. rX += <imm> (arithmetic operations with immediate operand); 4610 */ 4611 static int bpf_core_reloc_insn(struct bpf_program *prog, 4612 const struct bpf_field_reloc *relo, 4613 int relo_idx, 4614 const struct bpf_core_spec *local_spec, 4615 const struct bpf_core_spec *targ_spec) 4616 { 4617 __u32 orig_val, new_val; 4618 struct bpf_insn *insn; 4619 bool validate = true; 4620 int insn_idx, err; 4621 __u8 class; 4622 4623 if (relo->insn_off % sizeof(struct bpf_insn)) 4624 return -EINVAL; 4625 insn_idx = relo->insn_off / sizeof(struct bpf_insn); 4626 insn = &prog->insns[insn_idx]; 4627 class = BPF_CLASS(insn->code); 4628 4629 if (relo->kind == BPF_FIELD_EXISTS) { 4630 orig_val = 1; /* can't generate EXISTS relo w/o local field */ 4631 new_val = targ_spec ? 1 : 0; 4632 } else if (!targ_spec) { 4633 pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n", 4634 bpf_program__title(prog, false), relo_idx, insn_idx); 4635 insn->code = BPF_JMP | BPF_CALL; 4636 insn->dst_reg = 0; 4637 insn->src_reg = 0; 4638 insn->off = 0; 4639 /* if this instruction is reachable (not a dead code), 4640 * verifier will complain with the following message: 4641 * invalid func unknown#195896080 4642 */ 4643 insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */ 4644 return 0; 4645 } else { 4646 err = bpf_core_calc_field_relo(prog, relo, local_spec, 4647 &orig_val, &validate); 4648 if (err) 4649 return err; 4650 err = bpf_core_calc_field_relo(prog, relo, targ_spec, 4651 &new_val, NULL); 4652 if (err) 4653 return err; 4654 } 4655 4656 switch (class) { 4657 case BPF_ALU: 4658 case BPF_ALU64: 4659 if (BPF_SRC(insn->code) != BPF_K) 4660 return -EINVAL; 4661 if (validate && insn->imm != orig_val) { 4662 pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n", 4663 bpf_program__title(prog, false), relo_idx, 4664 insn_idx, insn->imm, orig_val, new_val); 4665 return -EINVAL; 4666 } 4667 orig_val = insn->imm; 4668 insn->imm = new_val; 4669 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n", 4670 bpf_program__title(prog, false), relo_idx, insn_idx, 4671 orig_val, new_val); 4672 break; 4673 case BPF_LDX: 4674 case BPF_ST: 4675 case BPF_STX: 4676 if (validate && insn->off != orig_val) { 4677 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LD/LDX/ST/STX) value: got %u, exp %u -> %u\n", 4678 bpf_program__title(prog, false), relo_idx, 4679 insn_idx, insn->off, orig_val, new_val); 4680 return -EINVAL; 4681 } 4682 if (new_val > SHRT_MAX) { 4683 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n", 4684 bpf_program__title(prog, false), relo_idx, 4685 insn_idx, new_val); 4686 return -ERANGE; 4687 } 4688 orig_val = insn->off; 4689 insn->off = new_val; 4690 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n", 4691 bpf_program__title(prog, false), relo_idx, insn_idx, 4692 orig_val, new_val); 4693 break; 4694 default: 4695 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n", 4696 bpf_program__title(prog, false), relo_idx, 4697 insn_idx, insn->code, insn->src_reg, insn->dst_reg, 4698 insn->off, insn->imm); 4699 return -EINVAL; 4700 } 4701 4702 return 0; 4703 } 4704 4705 /* Output spec definition in the format: 4706 * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>, 4707 * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b 4708 */ 4709 static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec) 4710 { 4711 const struct btf_type *t; 4712 const char *s; 4713 __u32 type_id; 4714 int i; 4715 4716 type_id = spec->spec[0].type_id; 4717 t = btf__type_by_id(spec->btf, type_id); 4718 s = btf__name_by_offset(spec->btf, t->name_off); 4719 libbpf_print(level, "[%u] %s + ", type_id, s); 4720 4721 for (i = 0; i < spec->raw_len; i++) 4722 libbpf_print(level, "%d%s", spec->raw_spec[i], 4723 i == spec->raw_len - 1 ? " => " : ":"); 4724 4725 libbpf_print(level, "%u.%u @ &x", 4726 spec->bit_offset / 8, spec->bit_offset % 8); 4727 4728 for (i = 0; i < spec->len; i++) { 4729 if (spec->spec[i].name) 4730 libbpf_print(level, ".%s", spec->spec[i].name); 4731 else 4732 libbpf_print(level, "[%u]", spec->spec[i].idx); 4733 } 4734 4735 } 4736 4737 static size_t bpf_core_hash_fn(const void *key, void *ctx) 4738 { 4739 return (size_t)key; 4740 } 4741 4742 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx) 4743 { 4744 return k1 == k2; 4745 } 4746 4747 static void *u32_as_hash_key(__u32 x) 4748 { 4749 return (void *)(uintptr_t)x; 4750 } 4751 4752 /* 4753 * CO-RE relocate single instruction. 4754 * 4755 * The outline and important points of the algorithm: 4756 * 1. For given local type, find corresponding candidate target types. 4757 * Candidate type is a type with the same "essential" name, ignoring 4758 * everything after last triple underscore (___). E.g., `sample`, 4759 * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates 4760 * for each other. Names with triple underscore are referred to as 4761 * "flavors" and are useful, among other things, to allow to 4762 * specify/support incompatible variations of the same kernel struct, which 4763 * might differ between different kernel versions and/or build 4764 * configurations. 4765 * 4766 * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C 4767 * converter, when deduplicated BTF of a kernel still contains more than 4768 * one different types with the same name. In that case, ___2, ___3, etc 4769 * are appended starting from second name conflict. But start flavors are 4770 * also useful to be defined "locally", in BPF program, to extract same 4771 * data from incompatible changes between different kernel 4772 * versions/configurations. For instance, to handle field renames between 4773 * kernel versions, one can use two flavors of the struct name with the 4774 * same common name and use conditional relocations to extract that field, 4775 * depending on target kernel version. 4776 * 2. For each candidate type, try to match local specification to this 4777 * candidate target type. Matching involves finding corresponding 4778 * high-level spec accessors, meaning that all named fields should match, 4779 * as well as all array accesses should be within the actual bounds. Also, 4780 * types should be compatible (see bpf_core_fields_are_compat for details). 4781 * 3. It is supported and expected that there might be multiple flavors 4782 * matching the spec. As long as all the specs resolve to the same set of 4783 * offsets across all candidates, there is no error. If there is any 4784 * ambiguity, CO-RE relocation will fail. This is necessary to accomodate 4785 * imprefection of BTF deduplication, which can cause slight duplication of 4786 * the same BTF type, if some directly or indirectly referenced (by 4787 * pointer) type gets resolved to different actual types in different 4788 * object files. If such situation occurs, deduplicated BTF will end up 4789 * with two (or more) structurally identical types, which differ only in 4790 * types they refer to through pointer. This should be OK in most cases and 4791 * is not an error. 4792 * 4. Candidate types search is performed by linearly scanning through all 4793 * types in target BTF. It is anticipated that this is overall more 4794 * efficient memory-wise and not significantly worse (if not better) 4795 * CPU-wise compared to prebuilding a map from all local type names to 4796 * a list of candidate type names. It's also sped up by caching resolved 4797 * list of matching candidates per each local "root" type ID, that has at 4798 * least one bpf_field_reloc associated with it. This list is shared 4799 * between multiple relocations for the same type ID and is updated as some 4800 * of the candidates are pruned due to structural incompatibility. 4801 */ 4802 static int bpf_core_reloc_field(struct bpf_program *prog, 4803 const struct bpf_field_reloc *relo, 4804 int relo_idx, 4805 const struct btf *local_btf, 4806 const struct btf *targ_btf, 4807 struct hashmap *cand_cache) 4808 { 4809 const char *prog_name = bpf_program__title(prog, false); 4810 struct bpf_core_spec local_spec, cand_spec, targ_spec; 4811 const void *type_key = u32_as_hash_key(relo->type_id); 4812 const struct btf_type *local_type, *cand_type; 4813 const char *local_name, *cand_name; 4814 struct ids_vec *cand_ids; 4815 __u32 local_id, cand_id; 4816 const char *spec_str; 4817 int i, j, err; 4818 4819 local_id = relo->type_id; 4820 local_type = btf__type_by_id(local_btf, local_id); 4821 if (!local_type) 4822 return -EINVAL; 4823 4824 local_name = btf__name_by_offset(local_btf, local_type->name_off); 4825 if (str_is_empty(local_name)) 4826 return -EINVAL; 4827 4828 spec_str = btf__name_by_offset(local_btf, relo->access_str_off); 4829 if (str_is_empty(spec_str)) 4830 return -EINVAL; 4831 4832 err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec); 4833 if (err) { 4834 pr_warn("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n", 4835 prog_name, relo_idx, local_id, local_name, spec_str, 4836 err); 4837 return -EINVAL; 4838 } 4839 4840 pr_debug("prog '%s': relo #%d: kind %d, spec is ", prog_name, relo_idx, 4841 relo->kind); 4842 bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec); 4843 libbpf_print(LIBBPF_DEBUG, "\n"); 4844 4845 if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) { 4846 cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf); 4847 if (IS_ERR(cand_ids)) { 4848 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld", 4849 prog_name, relo_idx, local_id, local_name, 4850 PTR_ERR(cand_ids)); 4851 return PTR_ERR(cand_ids); 4852 } 4853 err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL); 4854 if (err) { 4855 bpf_core_free_cands(cand_ids); 4856 return err; 4857 } 4858 } 4859 4860 for (i = 0, j = 0; i < cand_ids->len; i++) { 4861 cand_id = cand_ids->data[i]; 4862 cand_type = btf__type_by_id(targ_btf, cand_id); 4863 cand_name = btf__name_by_offset(targ_btf, cand_type->name_off); 4864 4865 err = bpf_core_spec_match(&local_spec, targ_btf, 4866 cand_id, &cand_spec); 4867 pr_debug("prog '%s': relo #%d: matching candidate #%d %s against spec ", 4868 prog_name, relo_idx, i, cand_name); 4869 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec); 4870 libbpf_print(LIBBPF_DEBUG, ": %d\n", err); 4871 if (err < 0) { 4872 pr_warn("prog '%s': relo #%d: matching error: %d\n", 4873 prog_name, relo_idx, err); 4874 return err; 4875 } 4876 if (err == 0) 4877 continue; 4878 4879 if (j == 0) { 4880 targ_spec = cand_spec; 4881 } else if (cand_spec.bit_offset != targ_spec.bit_offset) { 4882 /* if there are many candidates, they should all 4883 * resolve to the same bit offset 4884 */ 4885 pr_warn("prog '%s': relo #%d: offset ambiguity: %u != %u\n", 4886 prog_name, relo_idx, cand_spec.bit_offset, 4887 targ_spec.bit_offset); 4888 return -EINVAL; 4889 } 4890 4891 cand_ids->data[j++] = cand_spec.spec[0].type_id; 4892 } 4893 4894 /* 4895 * For BPF_FIELD_EXISTS relo or when used BPF program has field 4896 * existence checks or kernel version/config checks, it's expected 4897 * that we might not find any candidates. In this case, if field 4898 * wasn't found in any candidate, the list of candidates shouldn't 4899 * change at all, we'll just handle relocating appropriately, 4900 * depending on relo's kind. 4901 */ 4902 if (j > 0) 4903 cand_ids->len = j; 4904 4905 /* 4906 * If no candidates were found, it might be both a programmer error, 4907 * as well as expected case, depending whether instruction w/ 4908 * relocation is guarded in some way that makes it unreachable (dead 4909 * code) if relocation can't be resolved. This is handled in 4910 * bpf_core_reloc_insn() uniformly by replacing that instruction with 4911 * BPF helper call insn (using invalid helper ID). If that instruction 4912 * is indeed unreachable, then it will be ignored and eliminated by 4913 * verifier. If it was an error, then verifier will complain and point 4914 * to a specific instruction number in its log. 4915 */ 4916 if (j == 0) 4917 pr_debug("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n", 4918 prog_name, relo_idx, local_id, local_name, spec_str); 4919 4920 /* bpf_core_reloc_insn should know how to handle missing targ_spec */ 4921 err = bpf_core_reloc_insn(prog, relo, relo_idx, &local_spec, 4922 j ? &targ_spec : NULL); 4923 if (err) { 4924 pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n", 4925 prog_name, relo_idx, relo->insn_off, err); 4926 return -EINVAL; 4927 } 4928 4929 return 0; 4930 } 4931 4932 static int 4933 bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path) 4934 { 4935 const struct btf_ext_info_sec *sec; 4936 const struct bpf_field_reloc *rec; 4937 const struct btf_ext_info *seg; 4938 struct hashmap_entry *entry; 4939 struct hashmap *cand_cache = NULL; 4940 struct bpf_program *prog; 4941 struct btf *targ_btf; 4942 const char *sec_name; 4943 int i, err = 0; 4944 4945 if (targ_btf_path) 4946 targ_btf = btf__parse_elf(targ_btf_path, NULL); 4947 else 4948 targ_btf = libbpf_find_kernel_btf(); 4949 if (IS_ERR(targ_btf)) { 4950 pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf)); 4951 return PTR_ERR(targ_btf); 4952 } 4953 4954 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); 4955 if (IS_ERR(cand_cache)) { 4956 err = PTR_ERR(cand_cache); 4957 goto out; 4958 } 4959 4960 seg = &obj->btf_ext->field_reloc_info; 4961 for_each_btf_ext_sec(seg, sec) { 4962 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 4963 if (str_is_empty(sec_name)) { 4964 err = -EINVAL; 4965 goto out; 4966 } 4967 prog = bpf_object__find_program_by_title(obj, sec_name); 4968 if (!prog) { 4969 pr_warn("failed to find program '%s' for CO-RE offset relocation\n", 4970 sec_name); 4971 err = -EINVAL; 4972 goto out; 4973 } 4974 4975 pr_debug("prog '%s': performing %d CO-RE offset relocs\n", 4976 sec_name, sec->num_info); 4977 4978 for_each_btf_ext_rec(seg, sec, i, rec) { 4979 err = bpf_core_reloc_field(prog, rec, i, obj->btf, 4980 targ_btf, cand_cache); 4981 if (err) { 4982 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", 4983 sec_name, i, err); 4984 goto out; 4985 } 4986 } 4987 } 4988 4989 out: 4990 btf__free(targ_btf); 4991 if (!IS_ERR_OR_NULL(cand_cache)) { 4992 hashmap__for_each_entry(cand_cache, entry, i) { 4993 bpf_core_free_cands(entry->value); 4994 } 4995 hashmap__free(cand_cache); 4996 } 4997 return err; 4998 } 4999 5000 static int 5001 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) 5002 { 5003 int err = 0; 5004 5005 if (obj->btf_ext->field_reloc_info.len) 5006 err = bpf_core_reloc_fields(obj, targ_btf_path); 5007 5008 return err; 5009 } 5010 5011 static int 5012 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 5013 struct reloc_desc *relo) 5014 { 5015 struct bpf_insn *insn, *new_insn; 5016 struct bpf_program *text; 5017 size_t new_cnt; 5018 int err; 5019 5020 if (prog->idx != obj->efile.text_shndx && prog->main_prog_cnt == 0) { 5021 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 5022 if (!text) { 5023 pr_warn("no .text section found yet relo into text exist\n"); 5024 return -LIBBPF_ERRNO__RELOC; 5025 } 5026 new_cnt = prog->insns_cnt + text->insns_cnt; 5027 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 5028 if (!new_insn) { 5029 pr_warn("oom in prog realloc\n"); 5030 return -ENOMEM; 5031 } 5032 prog->insns = new_insn; 5033 5034 if (obj->btf_ext) { 5035 err = bpf_program_reloc_btf_ext(prog, obj, 5036 text->section_name, 5037 prog->insns_cnt); 5038 if (err) 5039 return err; 5040 } 5041 5042 memcpy(new_insn + prog->insns_cnt, text->insns, 5043 text->insns_cnt * sizeof(*insn)); 5044 prog->main_prog_cnt = prog->insns_cnt; 5045 prog->insns_cnt = new_cnt; 5046 pr_debug("added %zd insn from %s to prog %s\n", 5047 text->insns_cnt, text->section_name, 5048 prog->section_name); 5049 } 5050 5051 insn = &prog->insns[relo->insn_idx]; 5052 insn->imm += relo->sym_off / 8 + prog->main_prog_cnt - relo->insn_idx; 5053 return 0; 5054 } 5055 5056 static int 5057 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 5058 { 5059 int i, err; 5060 5061 if (!prog) 5062 return 0; 5063 5064 if (obj->btf_ext) { 5065 err = bpf_program_reloc_btf_ext(prog, obj, 5066 prog->section_name, 0); 5067 if (err) 5068 return err; 5069 } 5070 5071 if (!prog->reloc_desc) 5072 return 0; 5073 5074 for (i = 0; i < prog->nr_reloc; i++) { 5075 struct reloc_desc *relo = &prog->reloc_desc[i]; 5076 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 5077 struct extern_desc *ext; 5078 5079 if (relo->insn_idx + 1 >= (int)prog->insns_cnt) { 5080 pr_warn("relocation out of range: '%s'\n", 5081 prog->section_name); 5082 return -LIBBPF_ERRNO__RELOC; 5083 } 5084 5085 switch (relo->type) { 5086 case RELO_LD64: 5087 insn[0].src_reg = BPF_PSEUDO_MAP_FD; 5088 insn[0].imm = obj->maps[relo->map_idx].fd; 5089 break; 5090 case RELO_DATA: 5091 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 5092 insn[1].imm = insn[0].imm + relo->sym_off; 5093 insn[0].imm = obj->maps[relo->map_idx].fd; 5094 break; 5095 case RELO_EXTERN: 5096 ext = &obj->externs[relo->sym_off]; 5097 if (ext->type == EXT_KCFG) { 5098 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 5099 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; 5100 insn[1].imm = ext->kcfg.data_off; 5101 } else /* EXT_KSYM */ { 5102 insn[0].imm = (__u32)ext->ksym.addr; 5103 insn[1].imm = ext->ksym.addr >> 32; 5104 } 5105 break; 5106 case RELO_CALL: 5107 err = bpf_program__reloc_text(prog, obj, relo); 5108 if (err) 5109 return err; 5110 break; 5111 default: 5112 pr_warn("relo #%d: bad relo type %d\n", i, relo->type); 5113 return -EINVAL; 5114 } 5115 } 5116 5117 zfree(&prog->reloc_desc); 5118 prog->nr_reloc = 0; 5119 return 0; 5120 } 5121 5122 static int 5123 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) 5124 { 5125 struct bpf_program *prog; 5126 size_t i; 5127 int err; 5128 5129 if (obj->btf_ext) { 5130 err = bpf_object__relocate_core(obj, targ_btf_path); 5131 if (err) { 5132 pr_warn("failed to perform CO-RE relocations: %d\n", 5133 err); 5134 return err; 5135 } 5136 } 5137 /* ensure .text is relocated first, as it's going to be copied as-is 5138 * later for sub-program calls 5139 */ 5140 for (i = 0; i < obj->nr_programs; i++) { 5141 prog = &obj->programs[i]; 5142 if (prog->idx != obj->efile.text_shndx) 5143 continue; 5144 5145 err = bpf_program__relocate(prog, obj); 5146 if (err) { 5147 pr_warn("failed to relocate '%s'\n", prog->section_name); 5148 return err; 5149 } 5150 break; 5151 } 5152 /* now relocate everything but .text, which by now is relocated 5153 * properly, so we can copy raw sub-program instructions as is safely 5154 */ 5155 for (i = 0; i < obj->nr_programs; i++) { 5156 prog = &obj->programs[i]; 5157 if (prog->idx == obj->efile.text_shndx) 5158 continue; 5159 5160 err = bpf_program__relocate(prog, obj); 5161 if (err) { 5162 pr_warn("failed to relocate '%s'\n", prog->section_name); 5163 return err; 5164 } 5165 } 5166 return 0; 5167 } 5168 5169 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 5170 GElf_Shdr *shdr, Elf_Data *data); 5171 5172 static int bpf_object__collect_map_relos(struct bpf_object *obj, 5173 GElf_Shdr *shdr, Elf_Data *data) 5174 { 5175 int i, j, nrels, new_sz, ptr_sz = sizeof(void *); 5176 const struct btf_var_secinfo *vi = NULL; 5177 const struct btf_type *sec, *var, *def; 5178 const struct btf_member *member; 5179 struct bpf_map *map, *targ_map; 5180 const char *name, *mname; 5181 Elf_Data *symbols; 5182 unsigned int moff; 5183 GElf_Sym sym; 5184 GElf_Rel rel; 5185 void *tmp; 5186 5187 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) 5188 return -EINVAL; 5189 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); 5190 if (!sec) 5191 return -EINVAL; 5192 5193 symbols = obj->efile.symbols; 5194 nrels = shdr->sh_size / shdr->sh_entsize; 5195 for (i = 0; i < nrels; i++) { 5196 if (!gelf_getrel(data, i, &rel)) { 5197 pr_warn(".maps relo #%d: failed to get ELF relo\n", i); 5198 return -LIBBPF_ERRNO__FORMAT; 5199 } 5200 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 5201 pr_warn(".maps relo #%d: symbol %zx not found\n", 5202 i, (size_t)GELF_R_SYM(rel.r_info)); 5203 return -LIBBPF_ERRNO__FORMAT; 5204 } 5205 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 5206 sym.st_name) ? : "<?>"; 5207 if (sym.st_shndx != obj->efile.btf_maps_shndx) { 5208 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", 5209 i, name); 5210 return -LIBBPF_ERRNO__RELOC; 5211 } 5212 5213 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n", 5214 i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value, 5215 (size_t)rel.r_offset, sym.st_name, name); 5216 5217 for (j = 0; j < obj->nr_maps; j++) { 5218 map = &obj->maps[j]; 5219 if (map->sec_idx != obj->efile.btf_maps_shndx) 5220 continue; 5221 5222 vi = btf_var_secinfos(sec) + map->btf_var_idx; 5223 if (vi->offset <= rel.r_offset && 5224 rel.r_offset + sizeof(void *) <= vi->offset + vi->size) 5225 break; 5226 } 5227 if (j == obj->nr_maps) { 5228 pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n", 5229 i, name, (size_t)rel.r_offset); 5230 return -EINVAL; 5231 } 5232 5233 if (!bpf_map_type__is_map_in_map(map->def.type)) 5234 return -EINVAL; 5235 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && 5236 map->def.key_size != sizeof(int)) { 5237 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", 5238 i, map->name, sizeof(int)); 5239 return -EINVAL; 5240 } 5241 5242 targ_map = bpf_object__find_map_by_name(obj, name); 5243 if (!targ_map) 5244 return -ESRCH; 5245 5246 var = btf__type_by_id(obj->btf, vi->type); 5247 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 5248 if (btf_vlen(def) == 0) 5249 return -EINVAL; 5250 member = btf_members(def) + btf_vlen(def) - 1; 5251 mname = btf__name_by_offset(obj->btf, member->name_off); 5252 if (strcmp(mname, "values")) 5253 return -EINVAL; 5254 5255 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; 5256 if (rel.r_offset - vi->offset < moff) 5257 return -EINVAL; 5258 5259 moff = rel.r_offset - vi->offset - moff; 5260 if (moff % ptr_sz) 5261 return -EINVAL; 5262 moff /= ptr_sz; 5263 if (moff >= map->init_slots_sz) { 5264 new_sz = moff + 1; 5265 tmp = realloc(map->init_slots, new_sz * ptr_sz); 5266 if (!tmp) 5267 return -ENOMEM; 5268 map->init_slots = tmp; 5269 memset(map->init_slots + map->init_slots_sz, 0, 5270 (new_sz - map->init_slots_sz) * ptr_sz); 5271 map->init_slots_sz = new_sz; 5272 } 5273 map->init_slots[moff] = targ_map; 5274 5275 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n", 5276 i, map->name, moff, name); 5277 } 5278 5279 return 0; 5280 } 5281 5282 static int bpf_object__collect_reloc(struct bpf_object *obj) 5283 { 5284 int i, err; 5285 5286 if (!obj_elf_valid(obj)) { 5287 pr_warn("Internal error: elf object is closed\n"); 5288 return -LIBBPF_ERRNO__INTERNAL; 5289 } 5290 5291 for (i = 0; i < obj->efile.nr_reloc_sects; i++) { 5292 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr; 5293 Elf_Data *data = obj->efile.reloc_sects[i].data; 5294 int idx = shdr->sh_info; 5295 struct bpf_program *prog; 5296 5297 if (shdr->sh_type != SHT_REL) { 5298 pr_warn("internal error at %d\n", __LINE__); 5299 return -LIBBPF_ERRNO__INTERNAL; 5300 } 5301 5302 if (idx == obj->efile.st_ops_shndx) { 5303 err = bpf_object__collect_st_ops_relos(obj, shdr, data); 5304 } else if (idx == obj->efile.btf_maps_shndx) { 5305 err = bpf_object__collect_map_relos(obj, shdr, data); 5306 } else { 5307 prog = bpf_object__find_prog_by_idx(obj, idx); 5308 if (!prog) { 5309 pr_warn("relocation failed: no prog in section(%d)\n", idx); 5310 return -LIBBPF_ERRNO__RELOC; 5311 } 5312 err = bpf_program__collect_reloc(prog, shdr, data, obj); 5313 } 5314 if (err) 5315 return err; 5316 } 5317 return 0; 5318 } 5319 5320 static int 5321 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 5322 char *license, __u32 kern_version, int *pfd) 5323 { 5324 struct bpf_load_program_attr load_attr; 5325 char *cp, errmsg[STRERR_BUFSIZE]; 5326 size_t log_buf_size = 0; 5327 char *log_buf = NULL; 5328 int btf_fd, ret; 5329 5330 if (!insns || !insns_cnt) 5331 return -EINVAL; 5332 5333 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 5334 load_attr.prog_type = prog->type; 5335 /* old kernels might not support specifying expected_attach_type */ 5336 if (!prog->caps->exp_attach_type && prog->sec_def && 5337 prog->sec_def->is_exp_attach_type_optional) 5338 load_attr.expected_attach_type = 0; 5339 else 5340 load_attr.expected_attach_type = prog->expected_attach_type; 5341 if (prog->caps->name) 5342 load_attr.name = prog->name; 5343 load_attr.insns = insns; 5344 load_attr.insns_cnt = insns_cnt; 5345 load_attr.license = license; 5346 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 5347 prog->type == BPF_PROG_TYPE_LSM) { 5348 load_attr.attach_btf_id = prog->attach_btf_id; 5349 } else if (prog->type == BPF_PROG_TYPE_TRACING || 5350 prog->type == BPF_PROG_TYPE_EXT) { 5351 load_attr.attach_prog_fd = prog->attach_prog_fd; 5352 load_attr.attach_btf_id = prog->attach_btf_id; 5353 } else { 5354 load_attr.kern_version = kern_version; 5355 load_attr.prog_ifindex = prog->prog_ifindex; 5356 } 5357 /* if .BTF.ext was loaded, kernel supports associated BTF for prog */ 5358 if (prog->obj->btf_ext) 5359 btf_fd = bpf_object__btf_fd(prog->obj); 5360 else 5361 btf_fd = -1; 5362 load_attr.prog_btf_fd = btf_fd >= 0 ? btf_fd : 0; 5363 load_attr.func_info = prog->func_info; 5364 load_attr.func_info_rec_size = prog->func_info_rec_size; 5365 load_attr.func_info_cnt = prog->func_info_cnt; 5366 load_attr.line_info = prog->line_info; 5367 load_attr.line_info_rec_size = prog->line_info_rec_size; 5368 load_attr.line_info_cnt = prog->line_info_cnt; 5369 load_attr.log_level = prog->log_level; 5370 load_attr.prog_flags = prog->prog_flags; 5371 5372 retry_load: 5373 if (log_buf_size) { 5374 log_buf = malloc(log_buf_size); 5375 if (!log_buf) 5376 return -ENOMEM; 5377 5378 *log_buf = 0; 5379 } 5380 5381 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); 5382 5383 if (ret >= 0) { 5384 if (log_buf && load_attr.log_level) 5385 pr_debug("verifier log:\n%s", log_buf); 5386 *pfd = ret; 5387 ret = 0; 5388 goto out; 5389 } 5390 5391 if (!log_buf || errno == ENOSPC) { 5392 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, 5393 log_buf_size << 1); 5394 5395 free(log_buf); 5396 goto retry_load; 5397 } 5398 ret = -errno; 5399 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 5400 pr_warn("load bpf program failed: %s\n", cp); 5401 pr_perm_msg(ret); 5402 5403 if (log_buf && log_buf[0] != '\0') { 5404 ret = -LIBBPF_ERRNO__VERIFY; 5405 pr_warn("-- BEGIN DUMP LOG ---\n"); 5406 pr_warn("\n%s\n", log_buf); 5407 pr_warn("-- END LOG --\n"); 5408 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 5409 pr_warn("Program too large (%zu insns), at most %d insns\n", 5410 load_attr.insns_cnt, BPF_MAXINSNS); 5411 ret = -LIBBPF_ERRNO__PROG2BIG; 5412 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 5413 /* Wrong program type? */ 5414 int fd; 5415 5416 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 5417 load_attr.expected_attach_type = 0; 5418 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 5419 if (fd >= 0) { 5420 close(fd); 5421 ret = -LIBBPF_ERRNO__PROGTYPE; 5422 goto out; 5423 } 5424 } 5425 5426 out: 5427 free(log_buf); 5428 return ret; 5429 } 5430 5431 static int libbpf_find_attach_btf_id(struct bpf_program *prog); 5432 5433 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver) 5434 { 5435 int err = 0, fd, i, btf_id; 5436 5437 if ((prog->type == BPF_PROG_TYPE_TRACING || 5438 prog->type == BPF_PROG_TYPE_LSM || 5439 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) { 5440 btf_id = libbpf_find_attach_btf_id(prog); 5441 if (btf_id <= 0) 5442 return btf_id; 5443 prog->attach_btf_id = btf_id; 5444 } 5445 5446 if (prog->instances.nr < 0 || !prog->instances.fds) { 5447 if (prog->preprocessor) { 5448 pr_warn("Internal error: can't load program '%s'\n", 5449 prog->section_name); 5450 return -LIBBPF_ERRNO__INTERNAL; 5451 } 5452 5453 prog->instances.fds = malloc(sizeof(int)); 5454 if (!prog->instances.fds) { 5455 pr_warn("Not enough memory for BPF fds\n"); 5456 return -ENOMEM; 5457 } 5458 prog->instances.nr = 1; 5459 prog->instances.fds[0] = -1; 5460 } 5461 5462 if (!prog->preprocessor) { 5463 if (prog->instances.nr != 1) { 5464 pr_warn("Program '%s' is inconsistent: nr(%d) != 1\n", 5465 prog->section_name, prog->instances.nr); 5466 } 5467 err = load_program(prog, prog->insns, prog->insns_cnt, 5468 license, kern_ver, &fd); 5469 if (!err) 5470 prog->instances.fds[0] = fd; 5471 goto out; 5472 } 5473 5474 for (i = 0; i < prog->instances.nr; i++) { 5475 struct bpf_prog_prep_result result; 5476 bpf_program_prep_t preprocessor = prog->preprocessor; 5477 5478 memset(&result, 0, sizeof(result)); 5479 err = preprocessor(prog, i, prog->insns, 5480 prog->insns_cnt, &result); 5481 if (err) { 5482 pr_warn("Preprocessing the %dth instance of program '%s' failed\n", 5483 i, prog->section_name); 5484 goto out; 5485 } 5486 5487 if (!result.new_insn_ptr || !result.new_insn_cnt) { 5488 pr_debug("Skip loading the %dth instance of program '%s'\n", 5489 i, prog->section_name); 5490 prog->instances.fds[i] = -1; 5491 if (result.pfd) 5492 *result.pfd = -1; 5493 continue; 5494 } 5495 5496 err = load_program(prog, result.new_insn_ptr, 5497 result.new_insn_cnt, license, kern_ver, &fd); 5498 if (err) { 5499 pr_warn("Loading the %dth instance of program '%s' failed\n", 5500 i, prog->section_name); 5501 goto out; 5502 } 5503 5504 if (result.pfd) 5505 *result.pfd = fd; 5506 prog->instances.fds[i] = fd; 5507 } 5508 out: 5509 if (err) 5510 pr_warn("failed to load program '%s'\n", prog->section_name); 5511 zfree(&prog->insns); 5512 prog->insns_cnt = 0; 5513 return err; 5514 } 5515 5516 static bool bpf_program__is_function_storage(const struct bpf_program *prog, 5517 const struct bpf_object *obj) 5518 { 5519 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 5520 } 5521 5522 static int 5523 bpf_object__load_progs(struct bpf_object *obj, int log_level) 5524 { 5525 size_t i; 5526 int err; 5527 5528 for (i = 0; i < obj->nr_programs; i++) { 5529 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 5530 continue; 5531 obj->programs[i].log_level |= log_level; 5532 err = bpf_program__load(&obj->programs[i], 5533 obj->license, 5534 obj->kern_version); 5535 if (err) 5536 return err; 5537 } 5538 return 0; 5539 } 5540 5541 static const struct bpf_sec_def *find_sec_def(const char *sec_name); 5542 5543 static struct bpf_object * 5544 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, 5545 const struct bpf_object_open_opts *opts) 5546 { 5547 const char *obj_name, *kconfig; 5548 struct bpf_program *prog; 5549 struct bpf_object *obj; 5550 char tmp_name[64]; 5551 int err; 5552 5553 if (elf_version(EV_CURRENT) == EV_NONE) { 5554 pr_warn("failed to init libelf for %s\n", 5555 path ? : "(mem buf)"); 5556 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 5557 } 5558 5559 if (!OPTS_VALID(opts, bpf_object_open_opts)) 5560 return ERR_PTR(-EINVAL); 5561 5562 obj_name = OPTS_GET(opts, object_name, NULL); 5563 if (obj_buf) { 5564 if (!obj_name) { 5565 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 5566 (unsigned long)obj_buf, 5567 (unsigned long)obj_buf_sz); 5568 obj_name = tmp_name; 5569 } 5570 path = obj_name; 5571 pr_debug("loading object '%s' from buffer\n", obj_name); 5572 } 5573 5574 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); 5575 if (IS_ERR(obj)) 5576 return obj; 5577 5578 kconfig = OPTS_GET(opts, kconfig, NULL); 5579 if (kconfig) { 5580 obj->kconfig = strdup(kconfig); 5581 if (!obj->kconfig) 5582 return ERR_PTR(-ENOMEM); 5583 } 5584 5585 err = bpf_object__elf_init(obj); 5586 err = err ? : bpf_object__check_endianness(obj); 5587 err = err ? : bpf_object__elf_collect(obj); 5588 err = err ? : bpf_object__collect_externs(obj); 5589 err = err ? : bpf_object__finalize_btf(obj); 5590 err = err ? : bpf_object__init_maps(obj, opts); 5591 err = err ? : bpf_object__init_prog_names(obj); 5592 err = err ? : bpf_object__collect_reloc(obj); 5593 if (err) 5594 goto out; 5595 bpf_object__elf_finish(obj); 5596 5597 bpf_object__for_each_program(prog, obj) { 5598 prog->sec_def = find_sec_def(prog->section_name); 5599 if (!prog->sec_def) 5600 /* couldn't guess, but user might manually specify */ 5601 continue; 5602 5603 bpf_program__set_type(prog, prog->sec_def->prog_type); 5604 bpf_program__set_expected_attach_type(prog, 5605 prog->sec_def->expected_attach_type); 5606 5607 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING || 5608 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT) 5609 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); 5610 } 5611 5612 return obj; 5613 out: 5614 bpf_object__close(obj); 5615 return ERR_PTR(err); 5616 } 5617 5618 static struct bpf_object * 5619 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags) 5620 { 5621 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 5622 .relaxed_maps = flags & MAPS_RELAX_COMPAT, 5623 ); 5624 5625 /* param validation */ 5626 if (!attr->file) 5627 return NULL; 5628 5629 pr_debug("loading %s\n", attr->file); 5630 return __bpf_object__open(attr->file, NULL, 0, &opts); 5631 } 5632 5633 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 5634 { 5635 return __bpf_object__open_xattr(attr, 0); 5636 } 5637 5638 struct bpf_object *bpf_object__open(const char *path) 5639 { 5640 struct bpf_object_open_attr attr = { 5641 .file = path, 5642 .prog_type = BPF_PROG_TYPE_UNSPEC, 5643 }; 5644 5645 return bpf_object__open_xattr(&attr); 5646 } 5647 5648 struct bpf_object * 5649 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) 5650 { 5651 if (!path) 5652 return ERR_PTR(-EINVAL); 5653 5654 pr_debug("loading %s\n", path); 5655 5656 return __bpf_object__open(path, NULL, 0, opts); 5657 } 5658 5659 struct bpf_object * 5660 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, 5661 const struct bpf_object_open_opts *opts) 5662 { 5663 if (!obj_buf || obj_buf_sz == 0) 5664 return ERR_PTR(-EINVAL); 5665 5666 return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts); 5667 } 5668 5669 struct bpf_object * 5670 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, 5671 const char *name) 5672 { 5673 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, 5674 .object_name = name, 5675 /* wrong default, but backwards-compatible */ 5676 .relaxed_maps = true, 5677 ); 5678 5679 /* returning NULL is wrong, but backwards-compatible */ 5680 if (!obj_buf || obj_buf_sz == 0) 5681 return NULL; 5682 5683 return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts); 5684 } 5685 5686 int bpf_object__unload(struct bpf_object *obj) 5687 { 5688 size_t i; 5689 5690 if (!obj) 5691 return -EINVAL; 5692 5693 for (i = 0; i < obj->nr_maps; i++) { 5694 zclose(obj->maps[i].fd); 5695 if (obj->maps[i].st_ops) 5696 zfree(&obj->maps[i].st_ops->kern_vdata); 5697 } 5698 5699 for (i = 0; i < obj->nr_programs; i++) 5700 bpf_program__unload(&obj->programs[i]); 5701 5702 return 0; 5703 } 5704 5705 static int bpf_object__sanitize_maps(struct bpf_object *obj) 5706 { 5707 struct bpf_map *m; 5708 5709 bpf_object__for_each_map(m, obj) { 5710 if (!bpf_map__is_internal(m)) 5711 continue; 5712 if (!obj->caps.global_data) { 5713 pr_warn("kernel doesn't support global data\n"); 5714 return -ENOTSUP; 5715 } 5716 if (!obj->caps.array_mmap) 5717 m->def.map_flags ^= BPF_F_MMAPABLE; 5718 } 5719 5720 return 0; 5721 } 5722 5723 static int bpf_object__read_kallsyms_file(struct bpf_object *obj) 5724 { 5725 char sym_type, sym_name[500]; 5726 unsigned long long sym_addr; 5727 struct extern_desc *ext; 5728 int ret, err = 0; 5729 FILE *f; 5730 5731 f = fopen("/proc/kallsyms", "r"); 5732 if (!f) { 5733 err = -errno; 5734 pr_warn("failed to open /proc/kallsyms: %d\n", err); 5735 return err; 5736 } 5737 5738 while (true) { 5739 ret = fscanf(f, "%llx %c %499s%*[^\n]\n", 5740 &sym_addr, &sym_type, sym_name); 5741 if (ret == EOF && feof(f)) 5742 break; 5743 if (ret != 3) { 5744 pr_warn("failed to read kallasyms entry: %d\n", ret); 5745 err = -EINVAL; 5746 goto out; 5747 } 5748 5749 ext = find_extern_by_name(obj, sym_name); 5750 if (!ext || ext->type != EXT_KSYM) 5751 continue; 5752 5753 if (ext->is_set && ext->ksym.addr != sym_addr) { 5754 pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n", 5755 sym_name, ext->ksym.addr, sym_addr); 5756 err = -EINVAL; 5757 goto out; 5758 } 5759 if (!ext->is_set) { 5760 ext->is_set = true; 5761 ext->ksym.addr = sym_addr; 5762 pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr); 5763 } 5764 } 5765 5766 out: 5767 fclose(f); 5768 return err; 5769 } 5770 5771 static int bpf_object__resolve_externs(struct bpf_object *obj, 5772 const char *extra_kconfig) 5773 { 5774 bool need_config = false, need_kallsyms = false; 5775 struct extern_desc *ext; 5776 void *kcfg_data = NULL; 5777 int err, i; 5778 5779 if (obj->nr_extern == 0) 5780 return 0; 5781 5782 if (obj->kconfig_map_idx >= 0) 5783 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; 5784 5785 for (i = 0; i < obj->nr_extern; i++) { 5786 ext = &obj->externs[i]; 5787 5788 if (ext->type == EXT_KCFG && 5789 strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { 5790 void *ext_val = kcfg_data + ext->kcfg.data_off; 5791 __u32 kver = get_kernel_version(); 5792 5793 if (!kver) { 5794 pr_warn("failed to get kernel version\n"); 5795 return -EINVAL; 5796 } 5797 err = set_kcfg_value_num(ext, ext_val, kver); 5798 if (err) 5799 return err; 5800 pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver); 5801 } else if (ext->type == EXT_KCFG && 5802 strncmp(ext->name, "CONFIG_", 7) == 0) { 5803 need_config = true; 5804 } else if (ext->type == EXT_KSYM) { 5805 need_kallsyms = true; 5806 } else { 5807 pr_warn("unrecognized extern '%s'\n", ext->name); 5808 return -EINVAL; 5809 } 5810 } 5811 if (need_config && extra_kconfig) { 5812 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data); 5813 if (err) 5814 return -EINVAL; 5815 need_config = false; 5816 for (i = 0; i < obj->nr_extern; i++) { 5817 ext = &obj->externs[i]; 5818 if (ext->type == EXT_KCFG && !ext->is_set) { 5819 need_config = true; 5820 break; 5821 } 5822 } 5823 } 5824 if (need_config) { 5825 err = bpf_object__read_kconfig_file(obj, kcfg_data); 5826 if (err) 5827 return -EINVAL; 5828 } 5829 if (need_kallsyms) { 5830 err = bpf_object__read_kallsyms_file(obj); 5831 if (err) 5832 return -EINVAL; 5833 } 5834 for (i = 0; i < obj->nr_extern; i++) { 5835 ext = &obj->externs[i]; 5836 5837 if (!ext->is_set && !ext->is_weak) { 5838 pr_warn("extern %s (strong) not resolved\n", ext->name); 5839 return -ESRCH; 5840 } else if (!ext->is_set) { 5841 pr_debug("extern %s (weak) not resolved, defaulting to zero\n", 5842 ext->name); 5843 } 5844 } 5845 5846 return 0; 5847 } 5848 5849 int bpf_object__load_xattr(struct bpf_object_load_attr *attr) 5850 { 5851 struct bpf_object *obj; 5852 int err, i; 5853 5854 if (!attr) 5855 return -EINVAL; 5856 obj = attr->obj; 5857 if (!obj) 5858 return -EINVAL; 5859 5860 if (obj->loaded) { 5861 pr_warn("object should not be loaded twice\n"); 5862 return -EINVAL; 5863 } 5864 5865 obj->loaded = true; 5866 5867 err = bpf_object__probe_loading(obj); 5868 err = err ? : bpf_object__probe_caps(obj); 5869 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); 5870 err = err ? : bpf_object__sanitize_and_load_btf(obj); 5871 err = err ? : bpf_object__sanitize_maps(obj); 5872 err = err ? : bpf_object__load_vmlinux_btf(obj); 5873 err = err ? : bpf_object__init_kern_struct_ops_maps(obj); 5874 err = err ? : bpf_object__create_maps(obj); 5875 err = err ? : bpf_object__relocate(obj, attr->target_btf_path); 5876 err = err ? : bpf_object__load_progs(obj, attr->log_level); 5877 5878 btf__free(obj->btf_vmlinux); 5879 obj->btf_vmlinux = NULL; 5880 5881 if (err) 5882 goto out; 5883 5884 return 0; 5885 out: 5886 /* unpin any maps that were auto-pinned during load */ 5887 for (i = 0; i < obj->nr_maps; i++) 5888 if (obj->maps[i].pinned && !obj->maps[i].reused) 5889 bpf_map__unpin(&obj->maps[i], NULL); 5890 5891 bpf_object__unload(obj); 5892 pr_warn("failed to load object '%s'\n", obj->path); 5893 return err; 5894 } 5895 5896 int bpf_object__load(struct bpf_object *obj) 5897 { 5898 struct bpf_object_load_attr attr = { 5899 .obj = obj, 5900 }; 5901 5902 return bpf_object__load_xattr(&attr); 5903 } 5904 5905 static int make_parent_dir(const char *path) 5906 { 5907 char *cp, errmsg[STRERR_BUFSIZE]; 5908 char *dname, *dir; 5909 int err = 0; 5910 5911 dname = strdup(path); 5912 if (dname == NULL) 5913 return -ENOMEM; 5914 5915 dir = dirname(dname); 5916 if (mkdir(dir, 0700) && errno != EEXIST) 5917 err = -errno; 5918 5919 free(dname); 5920 if (err) { 5921 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 5922 pr_warn("failed to mkdir %s: %s\n", path, cp); 5923 } 5924 return err; 5925 } 5926 5927 static int check_path(const char *path) 5928 { 5929 char *cp, errmsg[STRERR_BUFSIZE]; 5930 struct statfs st_fs; 5931 char *dname, *dir; 5932 int err = 0; 5933 5934 if (path == NULL) 5935 return -EINVAL; 5936 5937 dname = strdup(path); 5938 if (dname == NULL) 5939 return -ENOMEM; 5940 5941 dir = dirname(dname); 5942 if (statfs(dir, &st_fs)) { 5943 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 5944 pr_warn("failed to statfs %s: %s\n", dir, cp); 5945 err = -errno; 5946 } 5947 free(dname); 5948 5949 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 5950 pr_warn("specified path %s is not on BPF FS\n", path); 5951 err = -EINVAL; 5952 } 5953 5954 return err; 5955 } 5956 5957 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 5958 int instance) 5959 { 5960 char *cp, errmsg[STRERR_BUFSIZE]; 5961 int err; 5962 5963 err = make_parent_dir(path); 5964 if (err) 5965 return err; 5966 5967 err = check_path(path); 5968 if (err) 5969 return err; 5970 5971 if (prog == NULL) { 5972 pr_warn("invalid program pointer\n"); 5973 return -EINVAL; 5974 } 5975 5976 if (instance < 0 || instance >= prog->instances.nr) { 5977 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 5978 instance, prog->section_name, prog->instances.nr); 5979 return -EINVAL; 5980 } 5981 5982 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 5983 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 5984 pr_warn("failed to pin program: %s\n", cp); 5985 return -errno; 5986 } 5987 pr_debug("pinned program '%s'\n", path); 5988 5989 return 0; 5990 } 5991 5992 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 5993 int instance) 5994 { 5995 int err; 5996 5997 err = check_path(path); 5998 if (err) 5999 return err; 6000 6001 if (prog == NULL) { 6002 pr_warn("invalid program pointer\n"); 6003 return -EINVAL; 6004 } 6005 6006 if (instance < 0 || instance >= prog->instances.nr) { 6007 pr_warn("invalid prog instance %d of prog %s (max %d)\n", 6008 instance, prog->section_name, prog->instances.nr); 6009 return -EINVAL; 6010 } 6011 6012 err = unlink(path); 6013 if (err != 0) 6014 return -errno; 6015 pr_debug("unpinned program '%s'\n", path); 6016 6017 return 0; 6018 } 6019 6020 int bpf_program__pin(struct bpf_program *prog, const char *path) 6021 { 6022 int i, err; 6023 6024 err = make_parent_dir(path); 6025 if (err) 6026 return err; 6027 6028 err = check_path(path); 6029 if (err) 6030 return err; 6031 6032 if (prog == NULL) { 6033 pr_warn("invalid program pointer\n"); 6034 return -EINVAL; 6035 } 6036 6037 if (prog->instances.nr <= 0) { 6038 pr_warn("no instances of prog %s to pin\n", 6039 prog->section_name); 6040 return -EINVAL; 6041 } 6042 6043 if (prog->instances.nr == 1) { 6044 /* don't create subdirs when pinning single instance */ 6045 return bpf_program__pin_instance(prog, path, 0); 6046 } 6047 6048 for (i = 0; i < prog->instances.nr; i++) { 6049 char buf[PATH_MAX]; 6050 int len; 6051 6052 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 6053 if (len < 0) { 6054 err = -EINVAL; 6055 goto err_unpin; 6056 } else if (len >= PATH_MAX) { 6057 err = -ENAMETOOLONG; 6058 goto err_unpin; 6059 } 6060 6061 err = bpf_program__pin_instance(prog, buf, i); 6062 if (err) 6063 goto err_unpin; 6064 } 6065 6066 return 0; 6067 6068 err_unpin: 6069 for (i = i - 1; i >= 0; i--) { 6070 char buf[PATH_MAX]; 6071 int len; 6072 6073 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 6074 if (len < 0) 6075 continue; 6076 else if (len >= PATH_MAX) 6077 continue; 6078 6079 bpf_program__unpin_instance(prog, buf, i); 6080 } 6081 6082 rmdir(path); 6083 6084 return err; 6085 } 6086 6087 int bpf_program__unpin(struct bpf_program *prog, const char *path) 6088 { 6089 int i, err; 6090 6091 err = check_path(path); 6092 if (err) 6093 return err; 6094 6095 if (prog == NULL) { 6096 pr_warn("invalid program pointer\n"); 6097 return -EINVAL; 6098 } 6099 6100 if (prog->instances.nr <= 0) { 6101 pr_warn("no instances of prog %s to pin\n", 6102 prog->section_name); 6103 return -EINVAL; 6104 } 6105 6106 if (prog->instances.nr == 1) { 6107 /* don't create subdirs when pinning single instance */ 6108 return bpf_program__unpin_instance(prog, path, 0); 6109 } 6110 6111 for (i = 0; i < prog->instances.nr; i++) { 6112 char buf[PATH_MAX]; 6113 int len; 6114 6115 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 6116 if (len < 0) 6117 return -EINVAL; 6118 else if (len >= PATH_MAX) 6119 return -ENAMETOOLONG; 6120 6121 err = bpf_program__unpin_instance(prog, buf, i); 6122 if (err) 6123 return err; 6124 } 6125 6126 err = rmdir(path); 6127 if (err) 6128 return -errno; 6129 6130 return 0; 6131 } 6132 6133 int bpf_map__pin(struct bpf_map *map, const char *path) 6134 { 6135 char *cp, errmsg[STRERR_BUFSIZE]; 6136 int err; 6137 6138 if (map == NULL) { 6139 pr_warn("invalid map pointer\n"); 6140 return -EINVAL; 6141 } 6142 6143 if (map->pin_path) { 6144 if (path && strcmp(path, map->pin_path)) { 6145 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 6146 bpf_map__name(map), map->pin_path, path); 6147 return -EINVAL; 6148 } else if (map->pinned) { 6149 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", 6150 bpf_map__name(map), map->pin_path); 6151 return 0; 6152 } 6153 } else { 6154 if (!path) { 6155 pr_warn("missing a path to pin map '%s' at\n", 6156 bpf_map__name(map)); 6157 return -EINVAL; 6158 } else if (map->pinned) { 6159 pr_warn("map '%s' already pinned\n", bpf_map__name(map)); 6160 return -EEXIST; 6161 } 6162 6163 map->pin_path = strdup(path); 6164 if (!map->pin_path) { 6165 err = -errno; 6166 goto out_err; 6167 } 6168 } 6169 6170 err = make_parent_dir(map->pin_path); 6171 if (err) 6172 return err; 6173 6174 err = check_path(map->pin_path); 6175 if (err) 6176 return err; 6177 6178 if (bpf_obj_pin(map->fd, map->pin_path)) { 6179 err = -errno; 6180 goto out_err; 6181 } 6182 6183 map->pinned = true; 6184 pr_debug("pinned map '%s'\n", map->pin_path); 6185 6186 return 0; 6187 6188 out_err: 6189 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 6190 pr_warn("failed to pin map: %s\n", cp); 6191 return err; 6192 } 6193 6194 int bpf_map__unpin(struct bpf_map *map, const char *path) 6195 { 6196 int err; 6197 6198 if (map == NULL) { 6199 pr_warn("invalid map pointer\n"); 6200 return -EINVAL; 6201 } 6202 6203 if (map->pin_path) { 6204 if (path && strcmp(path, map->pin_path)) { 6205 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 6206 bpf_map__name(map), map->pin_path, path); 6207 return -EINVAL; 6208 } 6209 path = map->pin_path; 6210 } else if (!path) { 6211 pr_warn("no path to unpin map '%s' from\n", 6212 bpf_map__name(map)); 6213 return -EINVAL; 6214 } 6215 6216 err = check_path(path); 6217 if (err) 6218 return err; 6219 6220 err = unlink(path); 6221 if (err != 0) 6222 return -errno; 6223 6224 map->pinned = false; 6225 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); 6226 6227 return 0; 6228 } 6229 6230 int bpf_map__set_pin_path(struct bpf_map *map, const char *path) 6231 { 6232 char *new = NULL; 6233 6234 if (path) { 6235 new = strdup(path); 6236 if (!new) 6237 return -errno; 6238 } 6239 6240 free(map->pin_path); 6241 map->pin_path = new; 6242 return 0; 6243 } 6244 6245 const char *bpf_map__get_pin_path(const struct bpf_map *map) 6246 { 6247 return map->pin_path; 6248 } 6249 6250 bool bpf_map__is_pinned(const struct bpf_map *map) 6251 { 6252 return map->pinned; 6253 } 6254 6255 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 6256 { 6257 struct bpf_map *map; 6258 int err; 6259 6260 if (!obj) 6261 return -ENOENT; 6262 6263 if (!obj->loaded) { 6264 pr_warn("object not yet loaded; load it first\n"); 6265 return -ENOENT; 6266 } 6267 6268 bpf_object__for_each_map(map, obj) { 6269 char *pin_path = NULL; 6270 char buf[PATH_MAX]; 6271 6272 if (path) { 6273 int len; 6274 6275 len = snprintf(buf, PATH_MAX, "%s/%s", path, 6276 bpf_map__name(map)); 6277 if (len < 0) { 6278 err = -EINVAL; 6279 goto err_unpin_maps; 6280 } else if (len >= PATH_MAX) { 6281 err = -ENAMETOOLONG; 6282 goto err_unpin_maps; 6283 } 6284 pin_path = buf; 6285 } else if (!map->pin_path) { 6286 continue; 6287 } 6288 6289 err = bpf_map__pin(map, pin_path); 6290 if (err) 6291 goto err_unpin_maps; 6292 } 6293 6294 return 0; 6295 6296 err_unpin_maps: 6297 while ((map = bpf_map__prev(map, obj))) { 6298 if (!map->pin_path) 6299 continue; 6300 6301 bpf_map__unpin(map, NULL); 6302 } 6303 6304 return err; 6305 } 6306 6307 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 6308 { 6309 struct bpf_map *map; 6310 int err; 6311 6312 if (!obj) 6313 return -ENOENT; 6314 6315 bpf_object__for_each_map(map, obj) { 6316 char *pin_path = NULL; 6317 char buf[PATH_MAX]; 6318 6319 if (path) { 6320 int len; 6321 6322 len = snprintf(buf, PATH_MAX, "%s/%s", path, 6323 bpf_map__name(map)); 6324 if (len < 0) 6325 return -EINVAL; 6326 else if (len >= PATH_MAX) 6327 return -ENAMETOOLONG; 6328 pin_path = buf; 6329 } else if (!map->pin_path) { 6330 continue; 6331 } 6332 6333 err = bpf_map__unpin(map, pin_path); 6334 if (err) 6335 return err; 6336 } 6337 6338 return 0; 6339 } 6340 6341 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 6342 { 6343 struct bpf_program *prog; 6344 int err; 6345 6346 if (!obj) 6347 return -ENOENT; 6348 6349 if (!obj->loaded) { 6350 pr_warn("object not yet loaded; load it first\n"); 6351 return -ENOENT; 6352 } 6353 6354 bpf_object__for_each_program(prog, obj) { 6355 char buf[PATH_MAX]; 6356 int len; 6357 6358 len = snprintf(buf, PATH_MAX, "%s/%s", path, 6359 prog->pin_name); 6360 if (len < 0) { 6361 err = -EINVAL; 6362 goto err_unpin_programs; 6363 } else if (len >= PATH_MAX) { 6364 err = -ENAMETOOLONG; 6365 goto err_unpin_programs; 6366 } 6367 6368 err = bpf_program__pin(prog, buf); 6369 if (err) 6370 goto err_unpin_programs; 6371 } 6372 6373 return 0; 6374 6375 err_unpin_programs: 6376 while ((prog = bpf_program__prev(prog, obj))) { 6377 char buf[PATH_MAX]; 6378 int len; 6379 6380 len = snprintf(buf, PATH_MAX, "%s/%s", path, 6381 prog->pin_name); 6382 if (len < 0) 6383 continue; 6384 else if (len >= PATH_MAX) 6385 continue; 6386 6387 bpf_program__unpin(prog, buf); 6388 } 6389 6390 return err; 6391 } 6392 6393 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 6394 { 6395 struct bpf_program *prog; 6396 int err; 6397 6398 if (!obj) 6399 return -ENOENT; 6400 6401 bpf_object__for_each_program(prog, obj) { 6402 char buf[PATH_MAX]; 6403 int len; 6404 6405 len = snprintf(buf, PATH_MAX, "%s/%s", path, 6406 prog->pin_name); 6407 if (len < 0) 6408 return -EINVAL; 6409 else if (len >= PATH_MAX) 6410 return -ENAMETOOLONG; 6411 6412 err = bpf_program__unpin(prog, buf); 6413 if (err) 6414 return err; 6415 } 6416 6417 return 0; 6418 } 6419 6420 int bpf_object__pin(struct bpf_object *obj, const char *path) 6421 { 6422 int err; 6423 6424 err = bpf_object__pin_maps(obj, path); 6425 if (err) 6426 return err; 6427 6428 err = bpf_object__pin_programs(obj, path); 6429 if (err) { 6430 bpf_object__unpin_maps(obj, path); 6431 return err; 6432 } 6433 6434 return 0; 6435 } 6436 6437 static void bpf_map__destroy(struct bpf_map *map) 6438 { 6439 if (map->clear_priv) 6440 map->clear_priv(map, map->priv); 6441 map->priv = NULL; 6442 map->clear_priv = NULL; 6443 6444 if (map->inner_map) { 6445 bpf_map__destroy(map->inner_map); 6446 zfree(&map->inner_map); 6447 } 6448 6449 zfree(&map->init_slots); 6450 map->init_slots_sz = 0; 6451 6452 if (map->mmaped) { 6453 munmap(map->mmaped, bpf_map_mmap_sz(map)); 6454 map->mmaped = NULL; 6455 } 6456 6457 if (map->st_ops) { 6458 zfree(&map->st_ops->data); 6459 zfree(&map->st_ops->progs); 6460 zfree(&map->st_ops->kern_func_off); 6461 zfree(&map->st_ops); 6462 } 6463 6464 zfree(&map->name); 6465 zfree(&map->pin_path); 6466 6467 if (map->fd >= 0) 6468 zclose(map->fd); 6469 } 6470 6471 void bpf_object__close(struct bpf_object *obj) 6472 { 6473 size_t i; 6474 6475 if (!obj) 6476 return; 6477 6478 if (obj->clear_priv) 6479 obj->clear_priv(obj, obj->priv); 6480 6481 bpf_object__elf_finish(obj); 6482 bpf_object__unload(obj); 6483 btf__free(obj->btf); 6484 btf_ext__free(obj->btf_ext); 6485 6486 for (i = 0; i < obj->nr_maps; i++) 6487 bpf_map__destroy(&obj->maps[i]); 6488 6489 zfree(&obj->kconfig); 6490 zfree(&obj->externs); 6491 obj->nr_extern = 0; 6492 6493 zfree(&obj->maps); 6494 obj->nr_maps = 0; 6495 6496 if (obj->programs && obj->nr_programs) { 6497 for (i = 0; i < obj->nr_programs; i++) 6498 bpf_program__exit(&obj->programs[i]); 6499 } 6500 zfree(&obj->programs); 6501 6502 list_del(&obj->list); 6503 free(obj); 6504 } 6505 6506 struct bpf_object * 6507 bpf_object__next(struct bpf_object *prev) 6508 { 6509 struct bpf_object *next; 6510 6511 if (!prev) 6512 next = list_first_entry(&bpf_objects_list, 6513 struct bpf_object, 6514 list); 6515 else 6516 next = list_next_entry(prev, list); 6517 6518 /* Empty list is noticed here so don't need checking on entry. */ 6519 if (&next->list == &bpf_objects_list) 6520 return NULL; 6521 6522 return next; 6523 } 6524 6525 const char *bpf_object__name(const struct bpf_object *obj) 6526 { 6527 return obj ? obj->name : ERR_PTR(-EINVAL); 6528 } 6529 6530 unsigned int bpf_object__kversion(const struct bpf_object *obj) 6531 { 6532 return obj ? obj->kern_version : 0; 6533 } 6534 6535 struct btf *bpf_object__btf(const struct bpf_object *obj) 6536 { 6537 return obj ? obj->btf : NULL; 6538 } 6539 6540 int bpf_object__btf_fd(const struct bpf_object *obj) 6541 { 6542 return obj->btf ? btf__fd(obj->btf) : -1; 6543 } 6544 6545 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 6546 bpf_object_clear_priv_t clear_priv) 6547 { 6548 if (obj->priv && obj->clear_priv) 6549 obj->clear_priv(obj, obj->priv); 6550 6551 obj->priv = priv; 6552 obj->clear_priv = clear_priv; 6553 return 0; 6554 } 6555 6556 void *bpf_object__priv(const struct bpf_object *obj) 6557 { 6558 return obj ? obj->priv : ERR_PTR(-EINVAL); 6559 } 6560 6561 static struct bpf_program * 6562 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 6563 bool forward) 6564 { 6565 size_t nr_programs = obj->nr_programs; 6566 ssize_t idx; 6567 6568 if (!nr_programs) 6569 return NULL; 6570 6571 if (!p) 6572 /* Iter from the beginning */ 6573 return forward ? &obj->programs[0] : 6574 &obj->programs[nr_programs - 1]; 6575 6576 if (p->obj != obj) { 6577 pr_warn("error: program handler doesn't match object\n"); 6578 return NULL; 6579 } 6580 6581 idx = (p - obj->programs) + (forward ? 1 : -1); 6582 if (idx >= obj->nr_programs || idx < 0) 6583 return NULL; 6584 return &obj->programs[idx]; 6585 } 6586 6587 struct bpf_program * 6588 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) 6589 { 6590 struct bpf_program *prog = prev; 6591 6592 do { 6593 prog = __bpf_program__iter(prog, obj, true); 6594 } while (prog && bpf_program__is_function_storage(prog, obj)); 6595 6596 return prog; 6597 } 6598 6599 struct bpf_program * 6600 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) 6601 { 6602 struct bpf_program *prog = next; 6603 6604 do { 6605 prog = __bpf_program__iter(prog, obj, false); 6606 } while (prog && bpf_program__is_function_storage(prog, obj)); 6607 6608 return prog; 6609 } 6610 6611 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 6612 bpf_program_clear_priv_t clear_priv) 6613 { 6614 if (prog->priv && prog->clear_priv) 6615 prog->clear_priv(prog, prog->priv); 6616 6617 prog->priv = priv; 6618 prog->clear_priv = clear_priv; 6619 return 0; 6620 } 6621 6622 void *bpf_program__priv(const struct bpf_program *prog) 6623 { 6624 return prog ? prog->priv : ERR_PTR(-EINVAL); 6625 } 6626 6627 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 6628 { 6629 prog->prog_ifindex = ifindex; 6630 } 6631 6632 const char *bpf_program__name(const struct bpf_program *prog) 6633 { 6634 return prog->name; 6635 } 6636 6637 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) 6638 { 6639 const char *title; 6640 6641 title = prog->section_name; 6642 if (needs_copy) { 6643 title = strdup(title); 6644 if (!title) { 6645 pr_warn("failed to strdup program title\n"); 6646 return ERR_PTR(-ENOMEM); 6647 } 6648 } 6649 6650 return title; 6651 } 6652 6653 int bpf_program__fd(const struct bpf_program *prog) 6654 { 6655 return bpf_program__nth_fd(prog, 0); 6656 } 6657 6658 size_t bpf_program__size(const struct bpf_program *prog) 6659 { 6660 return prog->insns_cnt * sizeof(struct bpf_insn); 6661 } 6662 6663 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 6664 bpf_program_prep_t prep) 6665 { 6666 int *instances_fds; 6667 6668 if (nr_instances <= 0 || !prep) 6669 return -EINVAL; 6670 6671 if (prog->instances.nr > 0 || prog->instances.fds) { 6672 pr_warn("Can't set pre-processor after loading\n"); 6673 return -EINVAL; 6674 } 6675 6676 instances_fds = malloc(sizeof(int) * nr_instances); 6677 if (!instances_fds) { 6678 pr_warn("alloc memory failed for fds\n"); 6679 return -ENOMEM; 6680 } 6681 6682 /* fill all fd with -1 */ 6683 memset(instances_fds, -1, sizeof(int) * nr_instances); 6684 6685 prog->instances.nr = nr_instances; 6686 prog->instances.fds = instances_fds; 6687 prog->preprocessor = prep; 6688 return 0; 6689 } 6690 6691 int bpf_program__nth_fd(const struct bpf_program *prog, int n) 6692 { 6693 int fd; 6694 6695 if (!prog) 6696 return -EINVAL; 6697 6698 if (n >= prog->instances.nr || n < 0) { 6699 pr_warn("Can't get the %dth fd from program %s: only %d instances\n", 6700 n, prog->section_name, prog->instances.nr); 6701 return -EINVAL; 6702 } 6703 6704 fd = prog->instances.fds[n]; 6705 if (fd < 0) { 6706 pr_warn("%dth instance of program '%s' is invalid\n", 6707 n, prog->section_name); 6708 return -ENOENT; 6709 } 6710 6711 return fd; 6712 } 6713 6714 enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog) 6715 { 6716 return prog->type; 6717 } 6718 6719 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 6720 { 6721 prog->type = type; 6722 } 6723 6724 static bool bpf_program__is_type(const struct bpf_program *prog, 6725 enum bpf_prog_type type) 6726 { 6727 return prog ? (prog->type == type) : false; 6728 } 6729 6730 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 6731 int bpf_program__set_##NAME(struct bpf_program *prog) \ 6732 { \ 6733 if (!prog) \ 6734 return -EINVAL; \ 6735 bpf_program__set_type(prog, TYPE); \ 6736 return 0; \ 6737 } \ 6738 \ 6739 bool bpf_program__is_##NAME(const struct bpf_program *prog) \ 6740 { \ 6741 return bpf_program__is_type(prog, TYPE); \ 6742 } \ 6743 6744 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 6745 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM); 6746 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 6747 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 6748 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 6749 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 6750 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 6751 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 6752 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 6753 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING); 6754 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS); 6755 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT); 6756 6757 enum bpf_attach_type 6758 bpf_program__get_expected_attach_type(struct bpf_program *prog) 6759 { 6760 return prog->expected_attach_type; 6761 } 6762 6763 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 6764 enum bpf_attach_type type) 6765 { 6766 prog->expected_attach_type = type; 6767 } 6768 6769 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional, \ 6770 attachable, attach_btf) \ 6771 { \ 6772 .sec = string, \ 6773 .len = sizeof(string) - 1, \ 6774 .prog_type = ptype, \ 6775 .expected_attach_type = eatype, \ 6776 .is_exp_attach_type_optional = eatype_optional, \ 6777 .is_attachable = attachable, \ 6778 .is_attach_btf = attach_btf, \ 6779 } 6780 6781 /* Programs that can NOT be attached. */ 6782 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0) 6783 6784 /* Programs that can be attached. */ 6785 #define BPF_APROG_SEC(string, ptype, atype) \ 6786 BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0) 6787 6788 /* Programs that must specify expected attach type at load time. */ 6789 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 6790 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0) 6791 6792 /* Programs that use BTF to identify attach point */ 6793 #define BPF_PROG_BTF(string, ptype, eatype) \ 6794 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1) 6795 6796 /* Programs that can be attached but attach type can't be identified by section 6797 * name. Kept for backward compatibility. 6798 */ 6799 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 6800 6801 #define SEC_DEF(sec_pfx, ptype, ...) { \ 6802 .sec = sec_pfx, \ 6803 .len = sizeof(sec_pfx) - 1, \ 6804 .prog_type = BPF_PROG_TYPE_##ptype, \ 6805 __VA_ARGS__ \ 6806 } 6807 6808 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, 6809 struct bpf_program *prog); 6810 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, 6811 struct bpf_program *prog); 6812 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, 6813 struct bpf_program *prog); 6814 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, 6815 struct bpf_program *prog); 6816 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec, 6817 struct bpf_program *prog); 6818 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec, 6819 struct bpf_program *prog); 6820 6821 static const struct bpf_sec_def section_defs[] = { 6822 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 6823 BPF_PROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT), 6824 SEC_DEF("kprobe/", KPROBE, 6825 .attach_fn = attach_kprobe), 6826 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE), 6827 SEC_DEF("kretprobe/", KPROBE, 6828 .attach_fn = attach_kprobe), 6829 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE), 6830 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 6831 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 6832 SEC_DEF("tracepoint/", TRACEPOINT, 6833 .attach_fn = attach_tp), 6834 SEC_DEF("tp/", TRACEPOINT, 6835 .attach_fn = attach_tp), 6836 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT, 6837 .attach_fn = attach_raw_tp), 6838 SEC_DEF("raw_tp/", RAW_TRACEPOINT, 6839 .attach_fn = attach_raw_tp), 6840 SEC_DEF("tp_btf/", TRACING, 6841 .expected_attach_type = BPF_TRACE_RAW_TP, 6842 .is_attach_btf = true, 6843 .attach_fn = attach_trace), 6844 SEC_DEF("fentry/", TRACING, 6845 .expected_attach_type = BPF_TRACE_FENTRY, 6846 .is_attach_btf = true, 6847 .attach_fn = attach_trace), 6848 SEC_DEF("fmod_ret/", TRACING, 6849 .expected_attach_type = BPF_MODIFY_RETURN, 6850 .is_attach_btf = true, 6851 .attach_fn = attach_trace), 6852 SEC_DEF("fexit/", TRACING, 6853 .expected_attach_type = BPF_TRACE_FEXIT, 6854 .is_attach_btf = true, 6855 .attach_fn = attach_trace), 6856 SEC_DEF("freplace/", EXT, 6857 .is_attach_btf = true, 6858 .attach_fn = attach_trace), 6859 SEC_DEF("lsm/", LSM, 6860 .is_attach_btf = true, 6861 .expected_attach_type = BPF_LSM_MAC, 6862 .attach_fn = attach_lsm), 6863 SEC_DEF("iter/", TRACING, 6864 .expected_attach_type = BPF_TRACE_ITER, 6865 .is_attach_btf = true, 6866 .attach_fn = attach_iter), 6867 BPF_EAPROG_SEC("xdp_devmap", BPF_PROG_TYPE_XDP, 6868 BPF_XDP_DEVMAP), 6869 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 6870 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 6871 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 6872 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 6873 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 6874 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 6875 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 6876 BPF_CGROUP_INET_INGRESS), 6877 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 6878 BPF_CGROUP_INET_EGRESS), 6879 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 6880 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 6881 BPF_CGROUP_INET_SOCK_CREATE), 6882 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 6883 BPF_CGROUP_INET4_POST_BIND), 6884 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 6885 BPF_CGROUP_INET6_POST_BIND), 6886 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 6887 BPF_CGROUP_DEVICE), 6888 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 6889 BPF_CGROUP_SOCK_OPS), 6890 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 6891 BPF_SK_SKB_STREAM_PARSER), 6892 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 6893 BPF_SK_SKB_STREAM_VERDICT), 6894 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 6895 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 6896 BPF_SK_MSG_VERDICT), 6897 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 6898 BPF_LIRC_MODE2), 6899 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 6900 BPF_FLOW_DISSECTOR), 6901 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6902 BPF_CGROUP_INET4_BIND), 6903 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6904 BPF_CGROUP_INET6_BIND), 6905 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6906 BPF_CGROUP_INET4_CONNECT), 6907 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6908 BPF_CGROUP_INET6_CONNECT), 6909 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6910 BPF_CGROUP_UDP4_SENDMSG), 6911 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6912 BPF_CGROUP_UDP6_SENDMSG), 6913 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6914 BPF_CGROUP_UDP4_RECVMSG), 6915 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6916 BPF_CGROUP_UDP6_RECVMSG), 6917 BPF_EAPROG_SEC("cgroup/getpeername4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6918 BPF_CGROUP_INET4_GETPEERNAME), 6919 BPF_EAPROG_SEC("cgroup/getpeername6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6920 BPF_CGROUP_INET6_GETPEERNAME), 6921 BPF_EAPROG_SEC("cgroup/getsockname4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6922 BPF_CGROUP_INET4_GETSOCKNAME), 6923 BPF_EAPROG_SEC("cgroup/getsockname6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 6924 BPF_CGROUP_INET6_GETSOCKNAME), 6925 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 6926 BPF_CGROUP_SYSCTL), 6927 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 6928 BPF_CGROUP_GETSOCKOPT), 6929 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 6930 BPF_CGROUP_SETSOCKOPT), 6931 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS), 6932 }; 6933 6934 #undef BPF_PROG_SEC_IMPL 6935 #undef BPF_PROG_SEC 6936 #undef BPF_APROG_SEC 6937 #undef BPF_EAPROG_SEC 6938 #undef BPF_APROG_COMPAT 6939 #undef SEC_DEF 6940 6941 #define MAX_TYPE_NAME_SIZE 32 6942 6943 static const struct bpf_sec_def *find_sec_def(const char *sec_name) 6944 { 6945 int i, n = ARRAY_SIZE(section_defs); 6946 6947 for (i = 0; i < n; i++) { 6948 if (strncmp(sec_name, 6949 section_defs[i].sec, section_defs[i].len)) 6950 continue; 6951 return §ion_defs[i]; 6952 } 6953 return NULL; 6954 } 6955 6956 static char *libbpf_get_type_names(bool attach_type) 6957 { 6958 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; 6959 char *buf; 6960 6961 buf = malloc(len); 6962 if (!buf) 6963 return NULL; 6964 6965 buf[0] = '\0'; 6966 /* Forge string buf with all available names */ 6967 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 6968 if (attach_type && !section_defs[i].is_attachable) 6969 continue; 6970 6971 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { 6972 free(buf); 6973 return NULL; 6974 } 6975 strcat(buf, " "); 6976 strcat(buf, section_defs[i].sec); 6977 } 6978 6979 return buf; 6980 } 6981 6982 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 6983 enum bpf_attach_type *expected_attach_type) 6984 { 6985 const struct bpf_sec_def *sec_def; 6986 char *type_names; 6987 6988 if (!name) 6989 return -EINVAL; 6990 6991 sec_def = find_sec_def(name); 6992 if (sec_def) { 6993 *prog_type = sec_def->prog_type; 6994 *expected_attach_type = sec_def->expected_attach_type; 6995 return 0; 6996 } 6997 6998 pr_debug("failed to guess program type from ELF section '%s'\n", name); 6999 type_names = libbpf_get_type_names(false); 7000 if (type_names != NULL) { 7001 pr_debug("supported section(type) names are:%s\n", type_names); 7002 free(type_names); 7003 } 7004 7005 return -ESRCH; 7006 } 7007 7008 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, 7009 size_t offset) 7010 { 7011 struct bpf_map *map; 7012 size_t i; 7013 7014 for (i = 0; i < obj->nr_maps; i++) { 7015 map = &obj->maps[i]; 7016 if (!bpf_map__is_struct_ops(map)) 7017 continue; 7018 if (map->sec_offset <= offset && 7019 offset - map->sec_offset < map->def.value_size) 7020 return map; 7021 } 7022 7023 return NULL; 7024 } 7025 7026 /* Collect the reloc from ELF and populate the st_ops->progs[] */ 7027 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 7028 GElf_Shdr *shdr, Elf_Data *data) 7029 { 7030 const struct btf_member *member; 7031 struct bpf_struct_ops *st_ops; 7032 struct bpf_program *prog; 7033 unsigned int shdr_idx; 7034 const struct btf *btf; 7035 struct bpf_map *map; 7036 Elf_Data *symbols; 7037 unsigned int moff; 7038 const char *name; 7039 __u32 member_idx; 7040 GElf_Sym sym; 7041 GElf_Rel rel; 7042 int i, nrels; 7043 7044 symbols = obj->efile.symbols; 7045 btf = obj->btf; 7046 nrels = shdr->sh_size / shdr->sh_entsize; 7047 for (i = 0; i < nrels; i++) { 7048 if (!gelf_getrel(data, i, &rel)) { 7049 pr_warn("struct_ops reloc: failed to get %d reloc\n", i); 7050 return -LIBBPF_ERRNO__FORMAT; 7051 } 7052 7053 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 7054 pr_warn("struct_ops reloc: symbol %zx not found\n", 7055 (size_t)GELF_R_SYM(rel.r_info)); 7056 return -LIBBPF_ERRNO__FORMAT; 7057 } 7058 7059 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 7060 sym.st_name) ? : "<?>"; 7061 map = find_struct_ops_map_by_offset(obj, rel.r_offset); 7062 if (!map) { 7063 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n", 7064 (size_t)rel.r_offset); 7065 return -EINVAL; 7066 } 7067 7068 moff = rel.r_offset - map->sec_offset; 7069 shdr_idx = sym.st_shndx; 7070 st_ops = map->st_ops; 7071 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", 7072 map->name, 7073 (long long)(rel.r_info >> 32), 7074 (long long)sym.st_value, 7075 shdr_idx, (size_t)rel.r_offset, 7076 map->sec_offset, sym.st_name, name); 7077 7078 if (shdr_idx >= SHN_LORESERVE) { 7079 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n", 7080 map->name, (size_t)rel.r_offset, shdr_idx); 7081 return -LIBBPF_ERRNO__RELOC; 7082 } 7083 7084 member = find_member_by_offset(st_ops->type, moff * 8); 7085 if (!member) { 7086 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", 7087 map->name, moff); 7088 return -EINVAL; 7089 } 7090 member_idx = member - btf_members(st_ops->type); 7091 name = btf__name_by_offset(btf, member->name_off); 7092 7093 if (!resolve_func_ptr(btf, member->type, NULL)) { 7094 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", 7095 map->name, name); 7096 return -EINVAL; 7097 } 7098 7099 prog = bpf_object__find_prog_by_idx(obj, shdr_idx); 7100 if (!prog) { 7101 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", 7102 map->name, shdr_idx, name); 7103 return -EINVAL; 7104 } 7105 7106 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 7107 const struct bpf_sec_def *sec_def; 7108 7109 sec_def = find_sec_def(prog->section_name); 7110 if (sec_def && 7111 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) { 7112 /* for pr_warn */ 7113 prog->type = sec_def->prog_type; 7114 goto invalid_prog; 7115 } 7116 7117 prog->type = BPF_PROG_TYPE_STRUCT_OPS; 7118 prog->attach_btf_id = st_ops->type_id; 7119 prog->expected_attach_type = member_idx; 7120 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS || 7121 prog->attach_btf_id != st_ops->type_id || 7122 prog->expected_attach_type != member_idx) { 7123 goto invalid_prog; 7124 } 7125 st_ops->progs[member_idx] = prog; 7126 } 7127 7128 return 0; 7129 7130 invalid_prog: 7131 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n", 7132 map->name, prog->name, prog->section_name, prog->type, 7133 prog->attach_btf_id, prog->expected_attach_type, name); 7134 return -EINVAL; 7135 } 7136 7137 #define BTF_TRACE_PREFIX "btf_trace_" 7138 #define BTF_LSM_PREFIX "bpf_lsm_" 7139 #define BTF_ITER_PREFIX "bpf_iter_" 7140 #define BTF_MAX_NAME_SIZE 128 7141 7142 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 7143 const char *name, __u32 kind) 7144 { 7145 char btf_type_name[BTF_MAX_NAME_SIZE]; 7146 int ret; 7147 7148 ret = snprintf(btf_type_name, sizeof(btf_type_name), 7149 "%s%s", prefix, name); 7150 /* snprintf returns the number of characters written excluding the 7151 * the terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it 7152 * indicates truncation. 7153 */ 7154 if (ret < 0 || ret >= sizeof(btf_type_name)) 7155 return -ENAMETOOLONG; 7156 return btf__find_by_name_kind(btf, btf_type_name, kind); 7157 } 7158 7159 static inline int __find_vmlinux_btf_id(struct btf *btf, const char *name, 7160 enum bpf_attach_type attach_type) 7161 { 7162 int err; 7163 7164 if (attach_type == BPF_TRACE_RAW_TP) 7165 err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name, 7166 BTF_KIND_TYPEDEF); 7167 else if (attach_type == BPF_LSM_MAC) 7168 err = find_btf_by_prefix_kind(btf, BTF_LSM_PREFIX, name, 7169 BTF_KIND_FUNC); 7170 else if (attach_type == BPF_TRACE_ITER) 7171 err = find_btf_by_prefix_kind(btf, BTF_ITER_PREFIX, name, 7172 BTF_KIND_FUNC); 7173 else 7174 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 7175 7176 if (err <= 0) 7177 pr_warn("%s is not found in vmlinux BTF\n", name); 7178 7179 return err; 7180 } 7181 7182 int libbpf_find_vmlinux_btf_id(const char *name, 7183 enum bpf_attach_type attach_type) 7184 { 7185 struct btf *btf; 7186 int err; 7187 7188 btf = libbpf_find_kernel_btf(); 7189 if (IS_ERR(btf)) { 7190 pr_warn("vmlinux BTF is not found\n"); 7191 return -EINVAL; 7192 } 7193 7194 err = __find_vmlinux_btf_id(btf, name, attach_type); 7195 btf__free(btf); 7196 return err; 7197 } 7198 7199 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) 7200 { 7201 struct bpf_prog_info_linear *info_linear; 7202 struct bpf_prog_info *info; 7203 struct btf *btf = NULL; 7204 int err = -EINVAL; 7205 7206 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0); 7207 if (IS_ERR_OR_NULL(info_linear)) { 7208 pr_warn("failed get_prog_info_linear for FD %d\n", 7209 attach_prog_fd); 7210 return -EINVAL; 7211 } 7212 info = &info_linear->info; 7213 if (!info->btf_id) { 7214 pr_warn("The target program doesn't have BTF\n"); 7215 goto out; 7216 } 7217 if (btf__get_from_id(info->btf_id, &btf)) { 7218 pr_warn("Failed to get BTF of the program\n"); 7219 goto out; 7220 } 7221 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 7222 btf__free(btf); 7223 if (err <= 0) { 7224 pr_warn("%s is not found in prog's BTF\n", name); 7225 goto out; 7226 } 7227 out: 7228 free(info_linear); 7229 return err; 7230 } 7231 7232 static int libbpf_find_attach_btf_id(struct bpf_program *prog) 7233 { 7234 enum bpf_attach_type attach_type = prog->expected_attach_type; 7235 __u32 attach_prog_fd = prog->attach_prog_fd; 7236 const char *name = prog->section_name; 7237 int i, err; 7238 7239 if (!name) 7240 return -EINVAL; 7241 7242 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 7243 if (!section_defs[i].is_attach_btf) 7244 continue; 7245 if (strncmp(name, section_defs[i].sec, section_defs[i].len)) 7246 continue; 7247 if (attach_prog_fd) 7248 err = libbpf_find_prog_btf_id(name + section_defs[i].len, 7249 attach_prog_fd); 7250 else 7251 err = __find_vmlinux_btf_id(prog->obj->btf_vmlinux, 7252 name + section_defs[i].len, 7253 attach_type); 7254 return err; 7255 } 7256 pr_warn("failed to identify btf_id based on ELF section name '%s'\n", name); 7257 return -ESRCH; 7258 } 7259 7260 int libbpf_attach_type_by_name(const char *name, 7261 enum bpf_attach_type *attach_type) 7262 { 7263 char *type_names; 7264 int i; 7265 7266 if (!name) 7267 return -EINVAL; 7268 7269 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 7270 if (strncmp(name, section_defs[i].sec, section_defs[i].len)) 7271 continue; 7272 if (!section_defs[i].is_attachable) 7273 return -EINVAL; 7274 *attach_type = section_defs[i].expected_attach_type; 7275 return 0; 7276 } 7277 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); 7278 type_names = libbpf_get_type_names(true); 7279 if (type_names != NULL) { 7280 pr_debug("attachable section(type) names are:%s\n", type_names); 7281 free(type_names); 7282 } 7283 7284 return -EINVAL; 7285 } 7286 7287 int bpf_map__fd(const struct bpf_map *map) 7288 { 7289 return map ? map->fd : -EINVAL; 7290 } 7291 7292 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) 7293 { 7294 return map ? &map->def : ERR_PTR(-EINVAL); 7295 } 7296 7297 const char *bpf_map__name(const struct bpf_map *map) 7298 { 7299 return map ? map->name : NULL; 7300 } 7301 7302 enum bpf_map_type bpf_map__type(const struct bpf_map *map) 7303 { 7304 return map->def.type; 7305 } 7306 7307 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type) 7308 { 7309 if (map->fd >= 0) 7310 return -EBUSY; 7311 map->def.type = type; 7312 return 0; 7313 } 7314 7315 __u32 bpf_map__map_flags(const struct bpf_map *map) 7316 { 7317 return map->def.map_flags; 7318 } 7319 7320 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags) 7321 { 7322 if (map->fd >= 0) 7323 return -EBUSY; 7324 map->def.map_flags = flags; 7325 return 0; 7326 } 7327 7328 __u32 bpf_map__numa_node(const struct bpf_map *map) 7329 { 7330 return map->numa_node; 7331 } 7332 7333 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node) 7334 { 7335 if (map->fd >= 0) 7336 return -EBUSY; 7337 map->numa_node = numa_node; 7338 return 0; 7339 } 7340 7341 __u32 bpf_map__key_size(const struct bpf_map *map) 7342 { 7343 return map->def.key_size; 7344 } 7345 7346 int bpf_map__set_key_size(struct bpf_map *map, __u32 size) 7347 { 7348 if (map->fd >= 0) 7349 return -EBUSY; 7350 map->def.key_size = size; 7351 return 0; 7352 } 7353 7354 __u32 bpf_map__value_size(const struct bpf_map *map) 7355 { 7356 return map->def.value_size; 7357 } 7358 7359 int bpf_map__set_value_size(struct bpf_map *map, __u32 size) 7360 { 7361 if (map->fd >= 0) 7362 return -EBUSY; 7363 map->def.value_size = size; 7364 return 0; 7365 } 7366 7367 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 7368 { 7369 return map ? map->btf_key_type_id : 0; 7370 } 7371 7372 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 7373 { 7374 return map ? map->btf_value_type_id : 0; 7375 } 7376 7377 int bpf_map__set_priv(struct bpf_map *map, void *priv, 7378 bpf_map_clear_priv_t clear_priv) 7379 { 7380 if (!map) 7381 return -EINVAL; 7382 7383 if (map->priv) { 7384 if (map->clear_priv) 7385 map->clear_priv(map, map->priv); 7386 } 7387 7388 map->priv = priv; 7389 map->clear_priv = clear_priv; 7390 return 0; 7391 } 7392 7393 void *bpf_map__priv(const struct bpf_map *map) 7394 { 7395 return map ? map->priv : ERR_PTR(-EINVAL); 7396 } 7397 7398 int bpf_map__set_initial_value(struct bpf_map *map, 7399 const void *data, size_t size) 7400 { 7401 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG || 7402 size != map->def.value_size || map->fd >= 0) 7403 return -EINVAL; 7404 7405 memcpy(map->mmaped, data, size); 7406 return 0; 7407 } 7408 7409 bool bpf_map__is_offload_neutral(const struct bpf_map *map) 7410 { 7411 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 7412 } 7413 7414 bool bpf_map__is_internal(const struct bpf_map *map) 7415 { 7416 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 7417 } 7418 7419 __u32 bpf_map__ifindex(const struct bpf_map *map) 7420 { 7421 return map->map_ifindex; 7422 } 7423 7424 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 7425 { 7426 if (map->fd >= 0) 7427 return -EBUSY; 7428 map->map_ifindex = ifindex; 7429 return 0; 7430 } 7431 7432 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 7433 { 7434 if (!bpf_map_type__is_map_in_map(map->def.type)) { 7435 pr_warn("error: unsupported map type\n"); 7436 return -EINVAL; 7437 } 7438 if (map->inner_map_fd != -1) { 7439 pr_warn("error: inner_map_fd already specified\n"); 7440 return -EINVAL; 7441 } 7442 map->inner_map_fd = fd; 7443 return 0; 7444 } 7445 7446 static struct bpf_map * 7447 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 7448 { 7449 ssize_t idx; 7450 struct bpf_map *s, *e; 7451 7452 if (!obj || !obj->maps) 7453 return NULL; 7454 7455 s = obj->maps; 7456 e = obj->maps + obj->nr_maps; 7457 7458 if ((m < s) || (m >= e)) { 7459 pr_warn("error in %s: map handler doesn't belong to object\n", 7460 __func__); 7461 return NULL; 7462 } 7463 7464 idx = (m - obj->maps) + i; 7465 if (idx >= obj->nr_maps || idx < 0) 7466 return NULL; 7467 return &obj->maps[idx]; 7468 } 7469 7470 struct bpf_map * 7471 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) 7472 { 7473 if (prev == NULL) 7474 return obj->maps; 7475 7476 return __bpf_map__iter(prev, obj, 1); 7477 } 7478 7479 struct bpf_map * 7480 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) 7481 { 7482 if (next == NULL) { 7483 if (!obj->nr_maps) 7484 return NULL; 7485 return obj->maps + obj->nr_maps - 1; 7486 } 7487 7488 return __bpf_map__iter(next, obj, -1); 7489 } 7490 7491 struct bpf_map * 7492 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 7493 { 7494 struct bpf_map *pos; 7495 7496 bpf_object__for_each_map(pos, obj) { 7497 if (pos->name && !strcmp(pos->name, name)) 7498 return pos; 7499 } 7500 return NULL; 7501 } 7502 7503 int 7504 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 7505 { 7506 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 7507 } 7508 7509 struct bpf_map * 7510 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 7511 { 7512 return ERR_PTR(-ENOTSUP); 7513 } 7514 7515 long libbpf_get_error(const void *ptr) 7516 { 7517 return PTR_ERR_OR_ZERO(ptr); 7518 } 7519 7520 int bpf_prog_load(const char *file, enum bpf_prog_type type, 7521 struct bpf_object **pobj, int *prog_fd) 7522 { 7523 struct bpf_prog_load_attr attr; 7524 7525 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 7526 attr.file = file; 7527 attr.prog_type = type; 7528 attr.expected_attach_type = 0; 7529 7530 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 7531 } 7532 7533 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 7534 struct bpf_object **pobj, int *prog_fd) 7535 { 7536 struct bpf_object_open_attr open_attr = {}; 7537 struct bpf_program *prog, *first_prog = NULL; 7538 struct bpf_object *obj; 7539 struct bpf_map *map; 7540 int err; 7541 7542 if (!attr) 7543 return -EINVAL; 7544 if (!attr->file) 7545 return -EINVAL; 7546 7547 open_attr.file = attr->file; 7548 open_attr.prog_type = attr->prog_type; 7549 7550 obj = bpf_object__open_xattr(&open_attr); 7551 if (IS_ERR_OR_NULL(obj)) 7552 return -ENOENT; 7553 7554 bpf_object__for_each_program(prog, obj) { 7555 enum bpf_attach_type attach_type = attr->expected_attach_type; 7556 /* 7557 * to preserve backwards compatibility, bpf_prog_load treats 7558 * attr->prog_type, if specified, as an override to whatever 7559 * bpf_object__open guessed 7560 */ 7561 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) { 7562 bpf_program__set_type(prog, attr->prog_type); 7563 bpf_program__set_expected_attach_type(prog, 7564 attach_type); 7565 } 7566 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) { 7567 /* 7568 * we haven't guessed from section name and user 7569 * didn't provide a fallback type, too bad... 7570 */ 7571 bpf_object__close(obj); 7572 return -EINVAL; 7573 } 7574 7575 prog->prog_ifindex = attr->ifindex; 7576 prog->log_level = attr->log_level; 7577 prog->prog_flags = attr->prog_flags; 7578 if (!first_prog) 7579 first_prog = prog; 7580 } 7581 7582 bpf_object__for_each_map(map, obj) { 7583 if (!bpf_map__is_offload_neutral(map)) 7584 map->map_ifindex = attr->ifindex; 7585 } 7586 7587 if (!first_prog) { 7588 pr_warn("object file doesn't contain bpf program\n"); 7589 bpf_object__close(obj); 7590 return -ENOENT; 7591 } 7592 7593 err = bpf_object__load(obj); 7594 if (err) { 7595 bpf_object__close(obj); 7596 return err; 7597 } 7598 7599 *pobj = obj; 7600 *prog_fd = bpf_program__fd(first_prog); 7601 return 0; 7602 } 7603 7604 struct bpf_link { 7605 int (*detach)(struct bpf_link *link); 7606 int (*destroy)(struct bpf_link *link); 7607 char *pin_path; /* NULL, if not pinned */ 7608 int fd; /* hook FD, -1 if not applicable */ 7609 bool disconnected; 7610 }; 7611 7612 /* Replace link's underlying BPF program with the new one */ 7613 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) 7614 { 7615 return bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL); 7616 } 7617 7618 /* Release "ownership" of underlying BPF resource (typically, BPF program 7619 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected 7620 * link, when destructed through bpf_link__destroy() call won't attempt to 7621 * detach/unregisted that BPF resource. This is useful in situations where, 7622 * say, attached BPF program has to outlive userspace program that attached it 7623 * in the system. Depending on type of BPF program, though, there might be 7624 * additional steps (like pinning BPF program in BPF FS) necessary to ensure 7625 * exit of userspace program doesn't trigger automatic detachment and clean up 7626 * inside the kernel. 7627 */ 7628 void bpf_link__disconnect(struct bpf_link *link) 7629 { 7630 link->disconnected = true; 7631 } 7632 7633 int bpf_link__destroy(struct bpf_link *link) 7634 { 7635 int err = 0; 7636 7637 if (!link) 7638 return 0; 7639 7640 if (!link->disconnected && link->detach) 7641 err = link->detach(link); 7642 if (link->destroy) 7643 link->destroy(link); 7644 if (link->pin_path) 7645 free(link->pin_path); 7646 free(link); 7647 7648 return err; 7649 } 7650 7651 int bpf_link__fd(const struct bpf_link *link) 7652 { 7653 return link->fd; 7654 } 7655 7656 const char *bpf_link__pin_path(const struct bpf_link *link) 7657 { 7658 return link->pin_path; 7659 } 7660 7661 static int bpf_link__detach_fd(struct bpf_link *link) 7662 { 7663 return close(link->fd); 7664 } 7665 7666 struct bpf_link *bpf_link__open(const char *path) 7667 { 7668 struct bpf_link *link; 7669 int fd; 7670 7671 fd = bpf_obj_get(path); 7672 if (fd < 0) { 7673 fd = -errno; 7674 pr_warn("failed to open link at %s: %d\n", path, fd); 7675 return ERR_PTR(fd); 7676 } 7677 7678 link = calloc(1, sizeof(*link)); 7679 if (!link) { 7680 close(fd); 7681 return ERR_PTR(-ENOMEM); 7682 } 7683 link->detach = &bpf_link__detach_fd; 7684 link->fd = fd; 7685 7686 link->pin_path = strdup(path); 7687 if (!link->pin_path) { 7688 bpf_link__destroy(link); 7689 return ERR_PTR(-ENOMEM); 7690 } 7691 7692 return link; 7693 } 7694 7695 int bpf_link__pin(struct bpf_link *link, const char *path) 7696 { 7697 int err; 7698 7699 if (link->pin_path) 7700 return -EBUSY; 7701 err = make_parent_dir(path); 7702 if (err) 7703 return err; 7704 err = check_path(path); 7705 if (err) 7706 return err; 7707 7708 link->pin_path = strdup(path); 7709 if (!link->pin_path) 7710 return -ENOMEM; 7711 7712 if (bpf_obj_pin(link->fd, link->pin_path)) { 7713 err = -errno; 7714 zfree(&link->pin_path); 7715 return err; 7716 } 7717 7718 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); 7719 return 0; 7720 } 7721 7722 int bpf_link__unpin(struct bpf_link *link) 7723 { 7724 int err; 7725 7726 if (!link->pin_path) 7727 return -EINVAL; 7728 7729 err = unlink(link->pin_path); 7730 if (err != 0) 7731 return -errno; 7732 7733 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); 7734 zfree(&link->pin_path); 7735 return 0; 7736 } 7737 7738 static int bpf_link__detach_perf_event(struct bpf_link *link) 7739 { 7740 int err; 7741 7742 err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0); 7743 if (err) 7744 err = -errno; 7745 7746 close(link->fd); 7747 return err; 7748 } 7749 7750 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, 7751 int pfd) 7752 { 7753 char errmsg[STRERR_BUFSIZE]; 7754 struct bpf_link *link; 7755 int prog_fd, err; 7756 7757 if (pfd < 0) { 7758 pr_warn("program '%s': invalid perf event FD %d\n", 7759 bpf_program__title(prog, false), pfd); 7760 return ERR_PTR(-EINVAL); 7761 } 7762 prog_fd = bpf_program__fd(prog); 7763 if (prog_fd < 0) { 7764 pr_warn("program '%s': can't attach BPF program w/o FD (did you load it?)\n", 7765 bpf_program__title(prog, false)); 7766 return ERR_PTR(-EINVAL); 7767 } 7768 7769 link = calloc(1, sizeof(*link)); 7770 if (!link) 7771 return ERR_PTR(-ENOMEM); 7772 link->detach = &bpf_link__detach_perf_event; 7773 link->fd = pfd; 7774 7775 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 7776 err = -errno; 7777 free(link); 7778 pr_warn("program '%s': failed to attach to pfd %d: %s\n", 7779 bpf_program__title(prog, false), pfd, 7780 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7781 return ERR_PTR(err); 7782 } 7783 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 7784 err = -errno; 7785 free(link); 7786 pr_warn("program '%s': failed to enable pfd %d: %s\n", 7787 bpf_program__title(prog, false), pfd, 7788 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7789 return ERR_PTR(err); 7790 } 7791 return link; 7792 } 7793 7794 /* 7795 * this function is expected to parse integer in the range of [0, 2^31-1] from 7796 * given file using scanf format string fmt. If actual parsed value is 7797 * negative, the result might be indistinguishable from error 7798 */ 7799 static int parse_uint_from_file(const char *file, const char *fmt) 7800 { 7801 char buf[STRERR_BUFSIZE]; 7802 int err, ret; 7803 FILE *f; 7804 7805 f = fopen(file, "r"); 7806 if (!f) { 7807 err = -errno; 7808 pr_debug("failed to open '%s': %s\n", file, 7809 libbpf_strerror_r(err, buf, sizeof(buf))); 7810 return err; 7811 } 7812 err = fscanf(f, fmt, &ret); 7813 if (err != 1) { 7814 err = err == EOF ? -EIO : -errno; 7815 pr_debug("failed to parse '%s': %s\n", file, 7816 libbpf_strerror_r(err, buf, sizeof(buf))); 7817 fclose(f); 7818 return err; 7819 } 7820 fclose(f); 7821 return ret; 7822 } 7823 7824 static int determine_kprobe_perf_type(void) 7825 { 7826 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 7827 7828 return parse_uint_from_file(file, "%d\n"); 7829 } 7830 7831 static int determine_uprobe_perf_type(void) 7832 { 7833 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 7834 7835 return parse_uint_from_file(file, "%d\n"); 7836 } 7837 7838 static int determine_kprobe_retprobe_bit(void) 7839 { 7840 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 7841 7842 return parse_uint_from_file(file, "config:%d\n"); 7843 } 7844 7845 static int determine_uprobe_retprobe_bit(void) 7846 { 7847 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 7848 7849 return parse_uint_from_file(file, "config:%d\n"); 7850 } 7851 7852 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 7853 uint64_t offset, int pid) 7854 { 7855 struct perf_event_attr attr = {}; 7856 char errmsg[STRERR_BUFSIZE]; 7857 int type, pfd, err; 7858 7859 type = uprobe ? determine_uprobe_perf_type() 7860 : determine_kprobe_perf_type(); 7861 if (type < 0) { 7862 pr_warn("failed to determine %s perf type: %s\n", 7863 uprobe ? "uprobe" : "kprobe", 7864 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 7865 return type; 7866 } 7867 if (retprobe) { 7868 int bit = uprobe ? determine_uprobe_retprobe_bit() 7869 : determine_kprobe_retprobe_bit(); 7870 7871 if (bit < 0) { 7872 pr_warn("failed to determine %s retprobe bit: %s\n", 7873 uprobe ? "uprobe" : "kprobe", 7874 libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); 7875 return bit; 7876 } 7877 attr.config |= 1 << bit; 7878 } 7879 attr.size = sizeof(attr); 7880 attr.type = type; 7881 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 7882 attr.config2 = offset; /* kprobe_addr or probe_offset */ 7883 7884 /* pid filter is meaningful only for uprobes */ 7885 pfd = syscall(__NR_perf_event_open, &attr, 7886 pid < 0 ? -1 : pid /* pid */, 7887 pid == -1 ? 0 : -1 /* cpu */, 7888 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 7889 if (pfd < 0) { 7890 err = -errno; 7891 pr_warn("%s perf_event_open() failed: %s\n", 7892 uprobe ? "uprobe" : "kprobe", 7893 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7894 return err; 7895 } 7896 return pfd; 7897 } 7898 7899 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, 7900 bool retprobe, 7901 const char *func_name) 7902 { 7903 char errmsg[STRERR_BUFSIZE]; 7904 struct bpf_link *link; 7905 int pfd, err; 7906 7907 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, 7908 0 /* offset */, -1 /* pid */); 7909 if (pfd < 0) { 7910 pr_warn("program '%s': failed to create %s '%s' perf event: %s\n", 7911 bpf_program__title(prog, false), 7912 retprobe ? "kretprobe" : "kprobe", func_name, 7913 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 7914 return ERR_PTR(pfd); 7915 } 7916 link = bpf_program__attach_perf_event(prog, pfd); 7917 if (IS_ERR(link)) { 7918 close(pfd); 7919 err = PTR_ERR(link); 7920 pr_warn("program '%s': failed to attach to %s '%s': %s\n", 7921 bpf_program__title(prog, false), 7922 retprobe ? "kretprobe" : "kprobe", func_name, 7923 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7924 return link; 7925 } 7926 return link; 7927 } 7928 7929 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, 7930 struct bpf_program *prog) 7931 { 7932 const char *func_name; 7933 bool retprobe; 7934 7935 func_name = bpf_program__title(prog, false) + sec->len; 7936 retprobe = strcmp(sec->sec, "kretprobe/") == 0; 7937 7938 return bpf_program__attach_kprobe(prog, retprobe, func_name); 7939 } 7940 7941 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, 7942 bool retprobe, pid_t pid, 7943 const char *binary_path, 7944 size_t func_offset) 7945 { 7946 char errmsg[STRERR_BUFSIZE]; 7947 struct bpf_link *link; 7948 int pfd, err; 7949 7950 pfd = perf_event_open_probe(true /* uprobe */, retprobe, 7951 binary_path, func_offset, pid); 7952 if (pfd < 0) { 7953 pr_warn("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 7954 bpf_program__title(prog, false), 7955 retprobe ? "uretprobe" : "uprobe", 7956 binary_path, func_offset, 7957 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 7958 return ERR_PTR(pfd); 7959 } 7960 link = bpf_program__attach_perf_event(prog, pfd); 7961 if (IS_ERR(link)) { 7962 close(pfd); 7963 err = PTR_ERR(link); 7964 pr_warn("program '%s': failed to attach to %s '%s:0x%zx': %s\n", 7965 bpf_program__title(prog, false), 7966 retprobe ? "uretprobe" : "uprobe", 7967 binary_path, func_offset, 7968 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 7969 return link; 7970 } 7971 return link; 7972 } 7973 7974 static int determine_tracepoint_id(const char *tp_category, 7975 const char *tp_name) 7976 { 7977 char file[PATH_MAX]; 7978 int ret; 7979 7980 ret = snprintf(file, sizeof(file), 7981 "/sys/kernel/debug/tracing/events/%s/%s/id", 7982 tp_category, tp_name); 7983 if (ret < 0) 7984 return -errno; 7985 if (ret >= sizeof(file)) { 7986 pr_debug("tracepoint %s/%s path is too long\n", 7987 tp_category, tp_name); 7988 return -E2BIG; 7989 } 7990 return parse_uint_from_file(file, "%d\n"); 7991 } 7992 7993 static int perf_event_open_tracepoint(const char *tp_category, 7994 const char *tp_name) 7995 { 7996 struct perf_event_attr attr = {}; 7997 char errmsg[STRERR_BUFSIZE]; 7998 int tp_id, pfd, err; 7999 8000 tp_id = determine_tracepoint_id(tp_category, tp_name); 8001 if (tp_id < 0) { 8002 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 8003 tp_category, tp_name, 8004 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 8005 return tp_id; 8006 } 8007 8008 attr.type = PERF_TYPE_TRACEPOINT; 8009 attr.size = sizeof(attr); 8010 attr.config = tp_id; 8011 8012 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 8013 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 8014 if (pfd < 0) { 8015 err = -errno; 8016 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", 8017 tp_category, tp_name, 8018 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 8019 return err; 8020 } 8021 return pfd; 8022 } 8023 8024 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, 8025 const char *tp_category, 8026 const char *tp_name) 8027 { 8028 char errmsg[STRERR_BUFSIZE]; 8029 struct bpf_link *link; 8030 int pfd, err; 8031 8032 pfd = perf_event_open_tracepoint(tp_category, tp_name); 8033 if (pfd < 0) { 8034 pr_warn("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 8035 bpf_program__title(prog, false), 8036 tp_category, tp_name, 8037 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 8038 return ERR_PTR(pfd); 8039 } 8040 link = bpf_program__attach_perf_event(prog, pfd); 8041 if (IS_ERR(link)) { 8042 close(pfd); 8043 err = PTR_ERR(link); 8044 pr_warn("program '%s': failed to attach to tracepoint '%s/%s': %s\n", 8045 bpf_program__title(prog, false), 8046 tp_category, tp_name, 8047 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 8048 return link; 8049 } 8050 return link; 8051 } 8052 8053 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, 8054 struct bpf_program *prog) 8055 { 8056 char *sec_name, *tp_cat, *tp_name; 8057 struct bpf_link *link; 8058 8059 sec_name = strdup(bpf_program__title(prog, false)); 8060 if (!sec_name) 8061 return ERR_PTR(-ENOMEM); 8062 8063 /* extract "tp/<category>/<name>" */ 8064 tp_cat = sec_name + sec->len; 8065 tp_name = strchr(tp_cat, '/'); 8066 if (!tp_name) { 8067 link = ERR_PTR(-EINVAL); 8068 goto out; 8069 } 8070 *tp_name = '\0'; 8071 tp_name++; 8072 8073 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); 8074 out: 8075 free(sec_name); 8076 return link; 8077 } 8078 8079 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, 8080 const char *tp_name) 8081 { 8082 char errmsg[STRERR_BUFSIZE]; 8083 struct bpf_link *link; 8084 int prog_fd, pfd; 8085 8086 prog_fd = bpf_program__fd(prog); 8087 if (prog_fd < 0) { 8088 pr_warn("program '%s': can't attach before loaded\n", 8089 bpf_program__title(prog, false)); 8090 return ERR_PTR(-EINVAL); 8091 } 8092 8093 link = calloc(1, sizeof(*link)); 8094 if (!link) 8095 return ERR_PTR(-ENOMEM); 8096 link->detach = &bpf_link__detach_fd; 8097 8098 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 8099 if (pfd < 0) { 8100 pfd = -errno; 8101 free(link); 8102 pr_warn("program '%s': failed to attach to raw tracepoint '%s': %s\n", 8103 bpf_program__title(prog, false), tp_name, 8104 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 8105 return ERR_PTR(pfd); 8106 } 8107 link->fd = pfd; 8108 return link; 8109 } 8110 8111 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, 8112 struct bpf_program *prog) 8113 { 8114 const char *tp_name = bpf_program__title(prog, false) + sec->len; 8115 8116 return bpf_program__attach_raw_tracepoint(prog, tp_name); 8117 } 8118 8119 /* Common logic for all BPF program types that attach to a btf_id */ 8120 static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog) 8121 { 8122 char errmsg[STRERR_BUFSIZE]; 8123 struct bpf_link *link; 8124 int prog_fd, pfd; 8125 8126 prog_fd = bpf_program__fd(prog); 8127 if (prog_fd < 0) { 8128 pr_warn("program '%s': can't attach before loaded\n", 8129 bpf_program__title(prog, false)); 8130 return ERR_PTR(-EINVAL); 8131 } 8132 8133 link = calloc(1, sizeof(*link)); 8134 if (!link) 8135 return ERR_PTR(-ENOMEM); 8136 link->detach = &bpf_link__detach_fd; 8137 8138 pfd = bpf_raw_tracepoint_open(NULL, prog_fd); 8139 if (pfd < 0) { 8140 pfd = -errno; 8141 free(link); 8142 pr_warn("program '%s': failed to attach: %s\n", 8143 bpf_program__title(prog, false), 8144 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 8145 return ERR_PTR(pfd); 8146 } 8147 link->fd = pfd; 8148 return (struct bpf_link *)link; 8149 } 8150 8151 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog) 8152 { 8153 return bpf_program__attach_btf_id(prog); 8154 } 8155 8156 struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog) 8157 { 8158 return bpf_program__attach_btf_id(prog); 8159 } 8160 8161 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, 8162 struct bpf_program *prog) 8163 { 8164 return bpf_program__attach_trace(prog); 8165 } 8166 8167 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec, 8168 struct bpf_program *prog) 8169 { 8170 return bpf_program__attach_lsm(prog); 8171 } 8172 8173 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec, 8174 struct bpf_program *prog) 8175 { 8176 return bpf_program__attach_iter(prog, NULL); 8177 } 8178 8179 static struct bpf_link * 8180 bpf_program__attach_fd(struct bpf_program *prog, int target_fd, 8181 const char *target_name) 8182 { 8183 enum bpf_attach_type attach_type; 8184 char errmsg[STRERR_BUFSIZE]; 8185 struct bpf_link *link; 8186 int prog_fd, link_fd; 8187 8188 prog_fd = bpf_program__fd(prog); 8189 if (prog_fd < 0) { 8190 pr_warn("program '%s': can't attach before loaded\n", 8191 bpf_program__title(prog, false)); 8192 return ERR_PTR(-EINVAL); 8193 } 8194 8195 link = calloc(1, sizeof(*link)); 8196 if (!link) 8197 return ERR_PTR(-ENOMEM); 8198 link->detach = &bpf_link__detach_fd; 8199 8200 attach_type = bpf_program__get_expected_attach_type(prog); 8201 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, NULL); 8202 if (link_fd < 0) { 8203 link_fd = -errno; 8204 free(link); 8205 pr_warn("program '%s': failed to attach to %s: %s\n", 8206 bpf_program__title(prog, false), target_name, 8207 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 8208 return ERR_PTR(link_fd); 8209 } 8210 link->fd = link_fd; 8211 return link; 8212 } 8213 8214 struct bpf_link * 8215 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd) 8216 { 8217 return bpf_program__attach_fd(prog, cgroup_fd, "cgroup"); 8218 } 8219 8220 struct bpf_link * 8221 bpf_program__attach_netns(struct bpf_program *prog, int netns_fd) 8222 { 8223 return bpf_program__attach_fd(prog, netns_fd, "netns"); 8224 } 8225 8226 struct bpf_link * 8227 bpf_program__attach_iter(struct bpf_program *prog, 8228 const struct bpf_iter_attach_opts *opts) 8229 { 8230 char errmsg[STRERR_BUFSIZE]; 8231 struct bpf_link *link; 8232 int prog_fd, link_fd; 8233 8234 if (!OPTS_VALID(opts, bpf_iter_attach_opts)) 8235 return ERR_PTR(-EINVAL); 8236 8237 prog_fd = bpf_program__fd(prog); 8238 if (prog_fd < 0) { 8239 pr_warn("program '%s': can't attach before loaded\n", 8240 bpf_program__title(prog, false)); 8241 return ERR_PTR(-EINVAL); 8242 } 8243 8244 link = calloc(1, sizeof(*link)); 8245 if (!link) 8246 return ERR_PTR(-ENOMEM); 8247 link->detach = &bpf_link__detach_fd; 8248 8249 link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_ITER, NULL); 8250 if (link_fd < 0) { 8251 link_fd = -errno; 8252 free(link); 8253 pr_warn("program '%s': failed to attach to iterator: %s\n", 8254 bpf_program__title(prog, false), 8255 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 8256 return ERR_PTR(link_fd); 8257 } 8258 link->fd = link_fd; 8259 return link; 8260 } 8261 8262 struct bpf_link *bpf_program__attach(struct bpf_program *prog) 8263 { 8264 const struct bpf_sec_def *sec_def; 8265 8266 sec_def = find_sec_def(bpf_program__title(prog, false)); 8267 if (!sec_def || !sec_def->attach_fn) 8268 return ERR_PTR(-ESRCH); 8269 8270 return sec_def->attach_fn(sec_def, prog); 8271 } 8272 8273 static int bpf_link__detach_struct_ops(struct bpf_link *link) 8274 { 8275 __u32 zero = 0; 8276 8277 if (bpf_map_delete_elem(link->fd, &zero)) 8278 return -errno; 8279 8280 return 0; 8281 } 8282 8283 struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map) 8284 { 8285 struct bpf_struct_ops *st_ops; 8286 struct bpf_link *link; 8287 __u32 i, zero = 0; 8288 int err; 8289 8290 if (!bpf_map__is_struct_ops(map) || map->fd == -1) 8291 return ERR_PTR(-EINVAL); 8292 8293 link = calloc(1, sizeof(*link)); 8294 if (!link) 8295 return ERR_PTR(-EINVAL); 8296 8297 st_ops = map->st_ops; 8298 for (i = 0; i < btf_vlen(st_ops->type); i++) { 8299 struct bpf_program *prog = st_ops->progs[i]; 8300 void *kern_data; 8301 int prog_fd; 8302 8303 if (!prog) 8304 continue; 8305 8306 prog_fd = bpf_program__fd(prog); 8307 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; 8308 *(unsigned long *)kern_data = prog_fd; 8309 } 8310 8311 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0); 8312 if (err) { 8313 err = -errno; 8314 free(link); 8315 return ERR_PTR(err); 8316 } 8317 8318 link->detach = bpf_link__detach_struct_ops; 8319 link->fd = map->fd; 8320 8321 return link; 8322 } 8323 8324 enum bpf_perf_event_ret 8325 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 8326 void **copy_mem, size_t *copy_size, 8327 bpf_perf_event_print_t fn, void *private_data) 8328 { 8329 struct perf_event_mmap_page *header = mmap_mem; 8330 __u64 data_head = ring_buffer_read_head(header); 8331 __u64 data_tail = header->data_tail; 8332 void *base = ((__u8 *)header) + page_size; 8333 int ret = LIBBPF_PERF_EVENT_CONT; 8334 struct perf_event_header *ehdr; 8335 size_t ehdr_size; 8336 8337 while (data_head != data_tail) { 8338 ehdr = base + (data_tail & (mmap_size - 1)); 8339 ehdr_size = ehdr->size; 8340 8341 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 8342 void *copy_start = ehdr; 8343 size_t len_first = base + mmap_size - copy_start; 8344 size_t len_secnd = ehdr_size - len_first; 8345 8346 if (*copy_size < ehdr_size) { 8347 free(*copy_mem); 8348 *copy_mem = malloc(ehdr_size); 8349 if (!*copy_mem) { 8350 *copy_size = 0; 8351 ret = LIBBPF_PERF_EVENT_ERROR; 8352 break; 8353 } 8354 *copy_size = ehdr_size; 8355 } 8356 8357 memcpy(*copy_mem, copy_start, len_first); 8358 memcpy(*copy_mem + len_first, base, len_secnd); 8359 ehdr = *copy_mem; 8360 } 8361 8362 ret = fn(ehdr, private_data); 8363 data_tail += ehdr_size; 8364 if (ret != LIBBPF_PERF_EVENT_CONT) 8365 break; 8366 } 8367 8368 ring_buffer_write_tail(header, data_tail); 8369 return ret; 8370 } 8371 8372 struct perf_buffer; 8373 8374 struct perf_buffer_params { 8375 struct perf_event_attr *attr; 8376 /* if event_cb is specified, it takes precendence */ 8377 perf_buffer_event_fn event_cb; 8378 /* sample_cb and lost_cb are higher-level common-case callbacks */ 8379 perf_buffer_sample_fn sample_cb; 8380 perf_buffer_lost_fn lost_cb; 8381 void *ctx; 8382 int cpu_cnt; 8383 int *cpus; 8384 int *map_keys; 8385 }; 8386 8387 struct perf_cpu_buf { 8388 struct perf_buffer *pb; 8389 void *base; /* mmap()'ed memory */ 8390 void *buf; /* for reconstructing segmented data */ 8391 size_t buf_size; 8392 int fd; 8393 int cpu; 8394 int map_key; 8395 }; 8396 8397 struct perf_buffer { 8398 perf_buffer_event_fn event_cb; 8399 perf_buffer_sample_fn sample_cb; 8400 perf_buffer_lost_fn lost_cb; 8401 void *ctx; /* passed into callbacks */ 8402 8403 size_t page_size; 8404 size_t mmap_size; 8405 struct perf_cpu_buf **cpu_bufs; 8406 struct epoll_event *events; 8407 int cpu_cnt; /* number of allocated CPU buffers */ 8408 int epoll_fd; /* perf event FD */ 8409 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 8410 }; 8411 8412 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 8413 struct perf_cpu_buf *cpu_buf) 8414 { 8415 if (!cpu_buf) 8416 return; 8417 if (cpu_buf->base && 8418 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 8419 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 8420 if (cpu_buf->fd >= 0) { 8421 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 8422 close(cpu_buf->fd); 8423 } 8424 free(cpu_buf->buf); 8425 free(cpu_buf); 8426 } 8427 8428 void perf_buffer__free(struct perf_buffer *pb) 8429 { 8430 int i; 8431 8432 if (!pb) 8433 return; 8434 if (pb->cpu_bufs) { 8435 for (i = 0; i < pb->cpu_cnt; i++) { 8436 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 8437 8438 if (!cpu_buf) 8439 continue; 8440 8441 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 8442 perf_buffer__free_cpu_buf(pb, cpu_buf); 8443 } 8444 free(pb->cpu_bufs); 8445 } 8446 if (pb->epoll_fd >= 0) 8447 close(pb->epoll_fd); 8448 free(pb->events); 8449 free(pb); 8450 } 8451 8452 static struct perf_cpu_buf * 8453 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 8454 int cpu, int map_key) 8455 { 8456 struct perf_cpu_buf *cpu_buf; 8457 char msg[STRERR_BUFSIZE]; 8458 int err; 8459 8460 cpu_buf = calloc(1, sizeof(*cpu_buf)); 8461 if (!cpu_buf) 8462 return ERR_PTR(-ENOMEM); 8463 8464 cpu_buf->pb = pb; 8465 cpu_buf->cpu = cpu; 8466 cpu_buf->map_key = map_key; 8467 8468 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 8469 -1, PERF_FLAG_FD_CLOEXEC); 8470 if (cpu_buf->fd < 0) { 8471 err = -errno; 8472 pr_warn("failed to open perf buffer event on cpu #%d: %s\n", 8473 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 8474 goto error; 8475 } 8476 8477 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 8478 PROT_READ | PROT_WRITE, MAP_SHARED, 8479 cpu_buf->fd, 0); 8480 if (cpu_buf->base == MAP_FAILED) { 8481 cpu_buf->base = NULL; 8482 err = -errno; 8483 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", 8484 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 8485 goto error; 8486 } 8487 8488 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 8489 err = -errno; 8490 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", 8491 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 8492 goto error; 8493 } 8494 8495 return cpu_buf; 8496 8497 error: 8498 perf_buffer__free_cpu_buf(pb, cpu_buf); 8499 return (struct perf_cpu_buf *)ERR_PTR(err); 8500 } 8501 8502 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 8503 struct perf_buffer_params *p); 8504 8505 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 8506 const struct perf_buffer_opts *opts) 8507 { 8508 struct perf_buffer_params p = {}; 8509 struct perf_event_attr attr = { 0, }; 8510 8511 attr.config = PERF_COUNT_SW_BPF_OUTPUT, 8512 attr.type = PERF_TYPE_SOFTWARE; 8513 attr.sample_type = PERF_SAMPLE_RAW; 8514 attr.sample_period = 1; 8515 attr.wakeup_events = 1; 8516 8517 p.attr = &attr; 8518 p.sample_cb = opts ? opts->sample_cb : NULL; 8519 p.lost_cb = opts ? opts->lost_cb : NULL; 8520 p.ctx = opts ? opts->ctx : NULL; 8521 8522 return __perf_buffer__new(map_fd, page_cnt, &p); 8523 } 8524 8525 struct perf_buffer * 8526 perf_buffer__new_raw(int map_fd, size_t page_cnt, 8527 const struct perf_buffer_raw_opts *opts) 8528 { 8529 struct perf_buffer_params p = {}; 8530 8531 p.attr = opts->attr; 8532 p.event_cb = opts->event_cb; 8533 p.ctx = opts->ctx; 8534 p.cpu_cnt = opts->cpu_cnt; 8535 p.cpus = opts->cpus; 8536 p.map_keys = opts->map_keys; 8537 8538 return __perf_buffer__new(map_fd, page_cnt, &p); 8539 } 8540 8541 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 8542 struct perf_buffer_params *p) 8543 { 8544 const char *online_cpus_file = "/sys/devices/system/cpu/online"; 8545 struct bpf_map_info map = {}; 8546 char msg[STRERR_BUFSIZE]; 8547 struct perf_buffer *pb; 8548 bool *online = NULL; 8549 __u32 map_info_len; 8550 int err, i, j, n; 8551 8552 if (page_cnt & (page_cnt - 1)) { 8553 pr_warn("page count should be power of two, but is %zu\n", 8554 page_cnt); 8555 return ERR_PTR(-EINVAL); 8556 } 8557 8558 map_info_len = sizeof(map); 8559 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); 8560 if (err) { 8561 err = -errno; 8562 pr_warn("failed to get map info for map FD %d: %s\n", 8563 map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); 8564 return ERR_PTR(err); 8565 } 8566 8567 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 8568 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 8569 map.name); 8570 return ERR_PTR(-EINVAL); 8571 } 8572 8573 pb = calloc(1, sizeof(*pb)); 8574 if (!pb) 8575 return ERR_PTR(-ENOMEM); 8576 8577 pb->event_cb = p->event_cb; 8578 pb->sample_cb = p->sample_cb; 8579 pb->lost_cb = p->lost_cb; 8580 pb->ctx = p->ctx; 8581 8582 pb->page_size = getpagesize(); 8583 pb->mmap_size = pb->page_size * page_cnt; 8584 pb->map_fd = map_fd; 8585 8586 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 8587 if (pb->epoll_fd < 0) { 8588 err = -errno; 8589 pr_warn("failed to create epoll instance: %s\n", 8590 libbpf_strerror_r(err, msg, sizeof(msg))); 8591 goto error; 8592 } 8593 8594 if (p->cpu_cnt > 0) { 8595 pb->cpu_cnt = p->cpu_cnt; 8596 } else { 8597 pb->cpu_cnt = libbpf_num_possible_cpus(); 8598 if (pb->cpu_cnt < 0) { 8599 err = pb->cpu_cnt; 8600 goto error; 8601 } 8602 if (map.max_entries < pb->cpu_cnt) 8603 pb->cpu_cnt = map.max_entries; 8604 } 8605 8606 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 8607 if (!pb->events) { 8608 err = -ENOMEM; 8609 pr_warn("failed to allocate events: out of memory\n"); 8610 goto error; 8611 } 8612 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 8613 if (!pb->cpu_bufs) { 8614 err = -ENOMEM; 8615 pr_warn("failed to allocate buffers: out of memory\n"); 8616 goto error; 8617 } 8618 8619 err = parse_cpu_mask_file(online_cpus_file, &online, &n); 8620 if (err) { 8621 pr_warn("failed to get online CPU mask: %d\n", err); 8622 goto error; 8623 } 8624 8625 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { 8626 struct perf_cpu_buf *cpu_buf; 8627 int cpu, map_key; 8628 8629 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 8630 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 8631 8632 /* in case user didn't explicitly requested particular CPUs to 8633 * be attached to, skip offline/not present CPUs 8634 */ 8635 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) 8636 continue; 8637 8638 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 8639 if (IS_ERR(cpu_buf)) { 8640 err = PTR_ERR(cpu_buf); 8641 goto error; 8642 } 8643 8644 pb->cpu_bufs[j] = cpu_buf; 8645 8646 err = bpf_map_update_elem(pb->map_fd, &map_key, 8647 &cpu_buf->fd, 0); 8648 if (err) { 8649 err = -errno; 8650 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 8651 cpu, map_key, cpu_buf->fd, 8652 libbpf_strerror_r(err, msg, sizeof(msg))); 8653 goto error; 8654 } 8655 8656 pb->events[j].events = EPOLLIN; 8657 pb->events[j].data.ptr = cpu_buf; 8658 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 8659 &pb->events[j]) < 0) { 8660 err = -errno; 8661 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 8662 cpu, cpu_buf->fd, 8663 libbpf_strerror_r(err, msg, sizeof(msg))); 8664 goto error; 8665 } 8666 j++; 8667 } 8668 pb->cpu_cnt = j; 8669 free(online); 8670 8671 return pb; 8672 8673 error: 8674 free(online); 8675 if (pb) 8676 perf_buffer__free(pb); 8677 return ERR_PTR(err); 8678 } 8679 8680 struct perf_sample_raw { 8681 struct perf_event_header header; 8682 uint32_t size; 8683 char data[]; 8684 }; 8685 8686 struct perf_sample_lost { 8687 struct perf_event_header header; 8688 uint64_t id; 8689 uint64_t lost; 8690 uint64_t sample_id; 8691 }; 8692 8693 static enum bpf_perf_event_ret 8694 perf_buffer__process_record(struct perf_event_header *e, void *ctx) 8695 { 8696 struct perf_cpu_buf *cpu_buf = ctx; 8697 struct perf_buffer *pb = cpu_buf->pb; 8698 void *data = e; 8699 8700 /* user wants full control over parsing perf event */ 8701 if (pb->event_cb) 8702 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 8703 8704 switch (e->type) { 8705 case PERF_RECORD_SAMPLE: { 8706 struct perf_sample_raw *s = data; 8707 8708 if (pb->sample_cb) 8709 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 8710 break; 8711 } 8712 case PERF_RECORD_LOST: { 8713 struct perf_sample_lost *s = data; 8714 8715 if (pb->lost_cb) 8716 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 8717 break; 8718 } 8719 default: 8720 pr_warn("unknown perf sample type %d\n", e->type); 8721 return LIBBPF_PERF_EVENT_ERROR; 8722 } 8723 return LIBBPF_PERF_EVENT_CONT; 8724 } 8725 8726 static int perf_buffer__process_records(struct perf_buffer *pb, 8727 struct perf_cpu_buf *cpu_buf) 8728 { 8729 enum bpf_perf_event_ret ret; 8730 8731 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, 8732 pb->page_size, &cpu_buf->buf, 8733 &cpu_buf->buf_size, 8734 perf_buffer__process_record, cpu_buf); 8735 if (ret != LIBBPF_PERF_EVENT_CONT) 8736 return ret; 8737 return 0; 8738 } 8739 8740 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 8741 { 8742 int i, cnt, err; 8743 8744 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 8745 for (i = 0; i < cnt; i++) { 8746 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 8747 8748 err = perf_buffer__process_records(pb, cpu_buf); 8749 if (err) { 8750 pr_warn("error while processing records: %d\n", err); 8751 return err; 8752 } 8753 } 8754 return cnt < 0 ? -errno : cnt; 8755 } 8756 8757 int perf_buffer__consume(struct perf_buffer *pb) 8758 { 8759 int i, err; 8760 8761 for (i = 0; i < pb->cpu_cnt; i++) { 8762 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 8763 8764 if (!cpu_buf) 8765 continue; 8766 8767 err = perf_buffer__process_records(pb, cpu_buf); 8768 if (err) { 8769 pr_warn("error while processing records: %d\n", err); 8770 return err; 8771 } 8772 } 8773 return 0; 8774 } 8775 8776 struct bpf_prog_info_array_desc { 8777 int array_offset; /* e.g. offset of jited_prog_insns */ 8778 int count_offset; /* e.g. offset of jited_prog_len */ 8779 int size_offset; /* > 0: offset of rec size, 8780 * < 0: fix size of -size_offset 8781 */ 8782 }; 8783 8784 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 8785 [BPF_PROG_INFO_JITED_INSNS] = { 8786 offsetof(struct bpf_prog_info, jited_prog_insns), 8787 offsetof(struct bpf_prog_info, jited_prog_len), 8788 -1, 8789 }, 8790 [BPF_PROG_INFO_XLATED_INSNS] = { 8791 offsetof(struct bpf_prog_info, xlated_prog_insns), 8792 offsetof(struct bpf_prog_info, xlated_prog_len), 8793 -1, 8794 }, 8795 [BPF_PROG_INFO_MAP_IDS] = { 8796 offsetof(struct bpf_prog_info, map_ids), 8797 offsetof(struct bpf_prog_info, nr_map_ids), 8798 -(int)sizeof(__u32), 8799 }, 8800 [BPF_PROG_INFO_JITED_KSYMS] = { 8801 offsetof(struct bpf_prog_info, jited_ksyms), 8802 offsetof(struct bpf_prog_info, nr_jited_ksyms), 8803 -(int)sizeof(__u64), 8804 }, 8805 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 8806 offsetof(struct bpf_prog_info, jited_func_lens), 8807 offsetof(struct bpf_prog_info, nr_jited_func_lens), 8808 -(int)sizeof(__u32), 8809 }, 8810 [BPF_PROG_INFO_FUNC_INFO] = { 8811 offsetof(struct bpf_prog_info, func_info), 8812 offsetof(struct bpf_prog_info, nr_func_info), 8813 offsetof(struct bpf_prog_info, func_info_rec_size), 8814 }, 8815 [BPF_PROG_INFO_LINE_INFO] = { 8816 offsetof(struct bpf_prog_info, line_info), 8817 offsetof(struct bpf_prog_info, nr_line_info), 8818 offsetof(struct bpf_prog_info, line_info_rec_size), 8819 }, 8820 [BPF_PROG_INFO_JITED_LINE_INFO] = { 8821 offsetof(struct bpf_prog_info, jited_line_info), 8822 offsetof(struct bpf_prog_info, nr_jited_line_info), 8823 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 8824 }, 8825 [BPF_PROG_INFO_PROG_TAGS] = { 8826 offsetof(struct bpf_prog_info, prog_tags), 8827 offsetof(struct bpf_prog_info, nr_prog_tags), 8828 -(int)sizeof(__u8) * BPF_TAG_SIZE, 8829 }, 8830 8831 }; 8832 8833 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, 8834 int offset) 8835 { 8836 __u32 *array = (__u32 *)info; 8837 8838 if (offset >= 0) 8839 return array[offset / sizeof(__u32)]; 8840 return -(int)offset; 8841 } 8842 8843 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, 8844 int offset) 8845 { 8846 __u64 *array = (__u64 *)info; 8847 8848 if (offset >= 0) 8849 return array[offset / sizeof(__u64)]; 8850 return -(int)offset; 8851 } 8852 8853 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 8854 __u32 val) 8855 { 8856 __u32 *array = (__u32 *)info; 8857 8858 if (offset >= 0) 8859 array[offset / sizeof(__u32)] = val; 8860 } 8861 8862 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 8863 __u64 val) 8864 { 8865 __u64 *array = (__u64 *)info; 8866 8867 if (offset >= 0) 8868 array[offset / sizeof(__u64)] = val; 8869 } 8870 8871 struct bpf_prog_info_linear * 8872 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 8873 { 8874 struct bpf_prog_info_linear *info_linear; 8875 struct bpf_prog_info info = {}; 8876 __u32 info_len = sizeof(info); 8877 __u32 data_len = 0; 8878 int i, err; 8879 void *ptr; 8880 8881 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 8882 return ERR_PTR(-EINVAL); 8883 8884 /* step 1: get array dimensions */ 8885 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 8886 if (err) { 8887 pr_debug("can't get prog info: %s", strerror(errno)); 8888 return ERR_PTR(-EFAULT); 8889 } 8890 8891 /* step 2: calculate total size of all arrays */ 8892 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 8893 bool include_array = (arrays & (1UL << i)) > 0; 8894 struct bpf_prog_info_array_desc *desc; 8895 __u32 count, size; 8896 8897 desc = bpf_prog_info_array_desc + i; 8898 8899 /* kernel is too old to support this field */ 8900 if (info_len < desc->array_offset + sizeof(__u32) || 8901 info_len < desc->count_offset + sizeof(__u32) || 8902 (desc->size_offset > 0 && info_len < desc->size_offset)) 8903 include_array = false; 8904 8905 if (!include_array) { 8906 arrays &= ~(1UL << i); /* clear the bit */ 8907 continue; 8908 } 8909 8910 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 8911 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 8912 8913 data_len += count * size; 8914 } 8915 8916 /* step 3: allocate continuous memory */ 8917 data_len = roundup(data_len, sizeof(__u64)); 8918 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 8919 if (!info_linear) 8920 return ERR_PTR(-ENOMEM); 8921 8922 /* step 4: fill data to info_linear->info */ 8923 info_linear->arrays = arrays; 8924 memset(&info_linear->info, 0, sizeof(info)); 8925 ptr = info_linear->data; 8926 8927 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 8928 struct bpf_prog_info_array_desc *desc; 8929 __u32 count, size; 8930 8931 if ((arrays & (1UL << i)) == 0) 8932 continue; 8933 8934 desc = bpf_prog_info_array_desc + i; 8935 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 8936 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 8937 bpf_prog_info_set_offset_u32(&info_linear->info, 8938 desc->count_offset, count); 8939 bpf_prog_info_set_offset_u32(&info_linear->info, 8940 desc->size_offset, size); 8941 bpf_prog_info_set_offset_u64(&info_linear->info, 8942 desc->array_offset, 8943 ptr_to_u64(ptr)); 8944 ptr += count * size; 8945 } 8946 8947 /* step 5: call syscall again to get required arrays */ 8948 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 8949 if (err) { 8950 pr_debug("can't get prog info: %s", strerror(errno)); 8951 free(info_linear); 8952 return ERR_PTR(-EFAULT); 8953 } 8954 8955 /* step 6: verify the data */ 8956 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 8957 struct bpf_prog_info_array_desc *desc; 8958 __u32 v1, v2; 8959 8960 if ((arrays & (1UL << i)) == 0) 8961 continue; 8962 8963 desc = bpf_prog_info_array_desc + i; 8964 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 8965 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 8966 desc->count_offset); 8967 if (v1 != v2) 8968 pr_warn("%s: mismatch in element count\n", __func__); 8969 8970 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 8971 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 8972 desc->size_offset); 8973 if (v1 != v2) 8974 pr_warn("%s: mismatch in rec size\n", __func__); 8975 } 8976 8977 /* step 7: update info_len and data_len */ 8978 info_linear->info_len = sizeof(struct bpf_prog_info); 8979 info_linear->data_len = data_len; 8980 8981 return info_linear; 8982 } 8983 8984 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 8985 { 8986 int i; 8987 8988 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 8989 struct bpf_prog_info_array_desc *desc; 8990 __u64 addr, offs; 8991 8992 if ((info_linear->arrays & (1UL << i)) == 0) 8993 continue; 8994 8995 desc = bpf_prog_info_array_desc + i; 8996 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 8997 desc->array_offset); 8998 offs = addr - ptr_to_u64(info_linear->data); 8999 bpf_prog_info_set_offset_u64(&info_linear->info, 9000 desc->array_offset, offs); 9001 } 9002 } 9003 9004 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 9005 { 9006 int i; 9007 9008 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 9009 struct bpf_prog_info_array_desc *desc; 9010 __u64 addr, offs; 9011 9012 if ((info_linear->arrays & (1UL << i)) == 0) 9013 continue; 9014 9015 desc = bpf_prog_info_array_desc + i; 9016 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 9017 desc->array_offset); 9018 addr = offs + ptr_to_u64(info_linear->data); 9019 bpf_prog_info_set_offset_u64(&info_linear->info, 9020 desc->array_offset, addr); 9021 } 9022 } 9023 9024 int bpf_program__set_attach_target(struct bpf_program *prog, 9025 int attach_prog_fd, 9026 const char *attach_func_name) 9027 { 9028 int btf_id; 9029 9030 if (!prog || attach_prog_fd < 0 || !attach_func_name) 9031 return -EINVAL; 9032 9033 if (attach_prog_fd) 9034 btf_id = libbpf_find_prog_btf_id(attach_func_name, 9035 attach_prog_fd); 9036 else 9037 btf_id = __find_vmlinux_btf_id(prog->obj->btf_vmlinux, 9038 attach_func_name, 9039 prog->expected_attach_type); 9040 9041 if (btf_id < 0) 9042 return btf_id; 9043 9044 prog->attach_btf_id = btf_id; 9045 prog->attach_prog_fd = attach_prog_fd; 9046 return 0; 9047 } 9048 9049 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) 9050 { 9051 int err = 0, n, len, start, end = -1; 9052 bool *tmp; 9053 9054 *mask = NULL; 9055 *mask_sz = 0; 9056 9057 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 9058 while (*s) { 9059 if (*s == ',' || *s == '\n') { 9060 s++; 9061 continue; 9062 } 9063 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); 9064 if (n <= 0 || n > 2) { 9065 pr_warn("Failed to get CPU range %s: %d\n", s, n); 9066 err = -EINVAL; 9067 goto cleanup; 9068 } else if (n == 1) { 9069 end = start; 9070 } 9071 if (start < 0 || start > end) { 9072 pr_warn("Invalid CPU range [%d,%d] in %s\n", 9073 start, end, s); 9074 err = -EINVAL; 9075 goto cleanup; 9076 } 9077 tmp = realloc(*mask, end + 1); 9078 if (!tmp) { 9079 err = -ENOMEM; 9080 goto cleanup; 9081 } 9082 *mask = tmp; 9083 memset(tmp + *mask_sz, 0, start - *mask_sz); 9084 memset(tmp + start, 1, end - start + 1); 9085 *mask_sz = end + 1; 9086 s += len; 9087 } 9088 if (!*mask_sz) { 9089 pr_warn("Empty CPU range\n"); 9090 return -EINVAL; 9091 } 9092 return 0; 9093 cleanup: 9094 free(*mask); 9095 *mask = NULL; 9096 return err; 9097 } 9098 9099 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) 9100 { 9101 int fd, err = 0, len; 9102 char buf[128]; 9103 9104 fd = open(fcpu, O_RDONLY); 9105 if (fd < 0) { 9106 err = -errno; 9107 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); 9108 return err; 9109 } 9110 len = read(fd, buf, sizeof(buf)); 9111 close(fd); 9112 if (len <= 0) { 9113 err = len ? -errno : -EINVAL; 9114 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); 9115 return err; 9116 } 9117 if (len >= sizeof(buf)) { 9118 pr_warn("CPU mask is too big in file %s\n", fcpu); 9119 return -E2BIG; 9120 } 9121 buf[len] = '\0'; 9122 9123 return parse_cpu_mask_str(buf, mask, mask_sz); 9124 } 9125 9126 int libbpf_num_possible_cpus(void) 9127 { 9128 static const char *fcpu = "/sys/devices/system/cpu/possible"; 9129 static int cpus; 9130 int err, n, i, tmp_cpus; 9131 bool *mask; 9132 9133 tmp_cpus = READ_ONCE(cpus); 9134 if (tmp_cpus > 0) 9135 return tmp_cpus; 9136 9137 err = parse_cpu_mask_file(fcpu, &mask, &n); 9138 if (err) 9139 return err; 9140 9141 tmp_cpus = 0; 9142 for (i = 0; i < n; i++) { 9143 if (mask[i]) 9144 tmp_cpus++; 9145 } 9146 free(mask); 9147 9148 WRITE_ONCE(cpus, tmp_cpus); 9149 return tmp_cpus; 9150 } 9151 9152 int bpf_object__open_skeleton(struct bpf_object_skeleton *s, 9153 const struct bpf_object_open_opts *opts) 9154 { 9155 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts, 9156 .object_name = s->name, 9157 ); 9158 struct bpf_object *obj; 9159 int i; 9160 9161 /* Attempt to preserve opts->object_name, unless overriden by user 9162 * explicitly. Overwriting object name for skeletons is discouraged, 9163 * as it breaks global data maps, because they contain object name 9164 * prefix as their own map name prefix. When skeleton is generated, 9165 * bpftool is making an assumption that this name will stay the same. 9166 */ 9167 if (opts) { 9168 memcpy(&skel_opts, opts, sizeof(*opts)); 9169 if (!opts->object_name) 9170 skel_opts.object_name = s->name; 9171 } 9172 9173 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); 9174 if (IS_ERR(obj)) { 9175 pr_warn("failed to initialize skeleton BPF object '%s': %ld\n", 9176 s->name, PTR_ERR(obj)); 9177 return PTR_ERR(obj); 9178 } 9179 9180 *s->obj = obj; 9181 9182 for (i = 0; i < s->map_cnt; i++) { 9183 struct bpf_map **map = s->maps[i].map; 9184 const char *name = s->maps[i].name; 9185 void **mmaped = s->maps[i].mmaped; 9186 9187 *map = bpf_object__find_map_by_name(obj, name); 9188 if (!*map) { 9189 pr_warn("failed to find skeleton map '%s'\n", name); 9190 return -ESRCH; 9191 } 9192 9193 /* externs shouldn't be pre-setup from user code */ 9194 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) 9195 *mmaped = (*map)->mmaped; 9196 } 9197 9198 for (i = 0; i < s->prog_cnt; i++) { 9199 struct bpf_program **prog = s->progs[i].prog; 9200 const char *name = s->progs[i].name; 9201 9202 *prog = bpf_object__find_program_by_name(obj, name); 9203 if (!*prog) { 9204 pr_warn("failed to find skeleton program '%s'\n", name); 9205 return -ESRCH; 9206 } 9207 } 9208 9209 return 0; 9210 } 9211 9212 int bpf_object__load_skeleton(struct bpf_object_skeleton *s) 9213 { 9214 int i, err; 9215 9216 err = bpf_object__load(*s->obj); 9217 if (err) { 9218 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); 9219 return err; 9220 } 9221 9222 for (i = 0; i < s->map_cnt; i++) { 9223 struct bpf_map *map = *s->maps[i].map; 9224 size_t mmap_sz = bpf_map_mmap_sz(map); 9225 int prot, map_fd = bpf_map__fd(map); 9226 void **mmaped = s->maps[i].mmaped; 9227 9228 if (!mmaped) 9229 continue; 9230 9231 if (!(map->def.map_flags & BPF_F_MMAPABLE)) { 9232 *mmaped = NULL; 9233 continue; 9234 } 9235 9236 if (map->def.map_flags & BPF_F_RDONLY_PROG) 9237 prot = PROT_READ; 9238 else 9239 prot = PROT_READ | PROT_WRITE; 9240 9241 /* Remap anonymous mmap()-ed "map initialization image" as 9242 * a BPF map-backed mmap()-ed memory, but preserving the same 9243 * memory address. This will cause kernel to change process' 9244 * page table to point to a different piece of kernel memory, 9245 * but from userspace point of view memory address (and its 9246 * contents, being identical at this point) will stay the 9247 * same. This mapping will be released by bpf_object__close() 9248 * as per normal clean up procedure, so we don't need to worry 9249 * about it from skeleton's clean up perspective. 9250 */ 9251 *mmaped = mmap(map->mmaped, mmap_sz, prot, 9252 MAP_SHARED | MAP_FIXED, map_fd, 0); 9253 if (*mmaped == MAP_FAILED) { 9254 err = -errno; 9255 *mmaped = NULL; 9256 pr_warn("failed to re-mmap() map '%s': %d\n", 9257 bpf_map__name(map), err); 9258 return err; 9259 } 9260 } 9261 9262 return 0; 9263 } 9264 9265 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) 9266 { 9267 int i; 9268 9269 for (i = 0; i < s->prog_cnt; i++) { 9270 struct bpf_program *prog = *s->progs[i].prog; 9271 struct bpf_link **link = s->progs[i].link; 9272 const struct bpf_sec_def *sec_def; 9273 const char *sec_name = bpf_program__title(prog, false); 9274 9275 sec_def = find_sec_def(sec_name); 9276 if (!sec_def || !sec_def->attach_fn) 9277 continue; 9278 9279 *link = sec_def->attach_fn(sec_def, prog); 9280 if (IS_ERR(*link)) { 9281 pr_warn("failed to auto-attach program '%s': %ld\n", 9282 bpf_program__name(prog), PTR_ERR(*link)); 9283 return PTR_ERR(*link); 9284 } 9285 } 9286 9287 return 0; 9288 } 9289 9290 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) 9291 { 9292 int i; 9293 9294 for (i = 0; i < s->prog_cnt; i++) { 9295 struct bpf_link **link = s->progs[i].link; 9296 9297 if (!IS_ERR_OR_NULL(*link)) 9298 bpf_link__destroy(*link); 9299 *link = NULL; 9300 } 9301 } 9302 9303 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) 9304 { 9305 if (s->progs) 9306 bpf_object__detach_skeleton(s); 9307 if (s->obj) 9308 bpf_object__close(*s->obj); 9309 free(s->maps); 9310 free(s->progs); 9311 free(s); 9312 } 9313