1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <string.h> 22 #include <unistd.h> 23 #include <fcntl.h> 24 #include <errno.h> 25 #include <asm/unistd.h> 26 #include <linux/err.h> 27 #include <linux/kernel.h> 28 #include <linux/bpf.h> 29 #include <linux/btf.h> 30 #include <linux/filter.h> 31 #include <linux/list.h> 32 #include <linux/limits.h> 33 #include <linux/perf_event.h> 34 #include <linux/ring_buffer.h> 35 #include <sys/ioctl.h> 36 #include <sys/stat.h> 37 #include <sys/types.h> 38 #include <sys/vfs.h> 39 #include <tools/libc_compat.h> 40 #include <libelf.h> 41 #include <gelf.h> 42 43 #include "libbpf.h" 44 #include "bpf.h" 45 #include "btf.h" 46 #include "str_error.h" 47 #include "libbpf_internal.h" 48 49 #ifndef EM_BPF 50 #define EM_BPF 247 51 #endif 52 53 #ifndef BPF_FS_MAGIC 54 #define BPF_FS_MAGIC 0xcafe4a11 55 #endif 56 57 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 58 * compilation if user enables corresponding warning. Disable it explicitly. 59 */ 60 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 61 62 #define __printf(a, b) __attribute__((format(printf, a, b))) 63 64 static int __base_pr(enum libbpf_print_level level, const char *format, 65 va_list args) 66 { 67 if (level == LIBBPF_DEBUG) 68 return 0; 69 70 return vfprintf(stderr, format, args); 71 } 72 73 static libbpf_print_fn_t __libbpf_pr = __base_pr; 74 75 void libbpf_set_print(libbpf_print_fn_t fn) 76 { 77 __libbpf_pr = fn; 78 } 79 80 __printf(2, 3) 81 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 82 { 83 va_list args; 84 85 if (!__libbpf_pr) 86 return; 87 88 va_start(args, format); 89 __libbpf_pr(level, format, args); 90 va_end(args); 91 } 92 93 #define STRERR_BUFSIZE 128 94 95 #define CHECK_ERR(action, err, out) do { \ 96 err = action; \ 97 if (err) \ 98 goto out; \ 99 } while(0) 100 101 102 /* Copied from tools/perf/util/util.h */ 103 #ifndef zfree 104 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 105 #endif 106 107 #ifndef zclose 108 # define zclose(fd) ({ \ 109 int ___err = 0; \ 110 if ((fd) >= 0) \ 111 ___err = close((fd)); \ 112 fd = -1; \ 113 ___err; }) 114 #endif 115 116 #ifdef HAVE_LIBELF_MMAP_SUPPORT 117 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 118 #else 119 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 120 #endif 121 122 static inline __u64 ptr_to_u64(const void *ptr) 123 { 124 return (__u64) (unsigned long) ptr; 125 } 126 127 struct bpf_capabilities { 128 /* v4.14: kernel support for program & map names. */ 129 __u32 name:1; 130 /* v5.2: kernel support for global data sections. */ 131 __u32 global_data:1; 132 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 133 __u32 btf_func:1; 134 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 135 __u32 btf_datasec:1; 136 }; 137 138 /* 139 * bpf_prog should be a better name but it has been used in 140 * linux/filter.h. 141 */ 142 struct bpf_program { 143 /* Index in elf obj file, for relocation use. */ 144 int idx; 145 char *name; 146 int prog_ifindex; 147 char *section_name; 148 /* section_name with / replaced by _; makes recursive pinning 149 * in bpf_object__pin_programs easier 150 */ 151 char *pin_name; 152 struct bpf_insn *insns; 153 size_t insns_cnt, main_prog_cnt; 154 enum bpf_prog_type type; 155 156 struct reloc_desc { 157 enum { 158 RELO_LD64, 159 RELO_CALL, 160 RELO_DATA, 161 } type; 162 int insn_idx; 163 union { 164 int map_idx; 165 int text_off; 166 }; 167 } *reloc_desc; 168 int nr_reloc; 169 int log_level; 170 171 struct { 172 int nr; 173 int *fds; 174 } instances; 175 bpf_program_prep_t preprocessor; 176 177 struct bpf_object *obj; 178 void *priv; 179 bpf_program_clear_priv_t clear_priv; 180 181 enum bpf_attach_type expected_attach_type; 182 int btf_fd; 183 void *func_info; 184 __u32 func_info_rec_size; 185 __u32 func_info_cnt; 186 187 struct bpf_capabilities *caps; 188 189 void *line_info; 190 __u32 line_info_rec_size; 191 __u32 line_info_cnt; 192 __u32 prog_flags; 193 }; 194 195 enum libbpf_map_type { 196 LIBBPF_MAP_UNSPEC, 197 LIBBPF_MAP_DATA, 198 LIBBPF_MAP_BSS, 199 LIBBPF_MAP_RODATA, 200 }; 201 202 static const char * const libbpf_type_to_btf_name[] = { 203 [LIBBPF_MAP_DATA] = ".data", 204 [LIBBPF_MAP_BSS] = ".bss", 205 [LIBBPF_MAP_RODATA] = ".rodata", 206 }; 207 208 struct bpf_map { 209 int fd; 210 char *name; 211 int sec_idx; 212 size_t sec_offset; 213 int map_ifindex; 214 int inner_map_fd; 215 struct bpf_map_def def; 216 __u32 btf_key_type_id; 217 __u32 btf_value_type_id; 218 void *priv; 219 bpf_map_clear_priv_t clear_priv; 220 enum libbpf_map_type libbpf_type; 221 }; 222 223 struct bpf_secdata { 224 void *rodata; 225 void *data; 226 }; 227 228 static LIST_HEAD(bpf_objects_list); 229 230 struct bpf_object { 231 char name[BPF_OBJ_NAME_LEN]; 232 char license[64]; 233 __u32 kern_version; 234 235 struct bpf_program *programs; 236 size_t nr_programs; 237 struct bpf_map *maps; 238 size_t nr_maps; 239 size_t maps_cap; 240 struct bpf_secdata sections; 241 242 bool loaded; 243 bool has_pseudo_calls; 244 245 /* 246 * Information when doing elf related work. Only valid if fd 247 * is valid. 248 */ 249 struct { 250 int fd; 251 void *obj_buf; 252 size_t obj_buf_sz; 253 Elf *elf; 254 GElf_Ehdr ehdr; 255 Elf_Data *symbols; 256 Elf_Data *data; 257 Elf_Data *rodata; 258 Elf_Data *bss; 259 size_t strtabidx; 260 struct { 261 GElf_Shdr shdr; 262 Elf_Data *data; 263 } *reloc; 264 int nr_reloc; 265 int maps_shndx; 266 int btf_maps_shndx; 267 int text_shndx; 268 int data_shndx; 269 int rodata_shndx; 270 int bss_shndx; 271 } efile; 272 /* 273 * All loaded bpf_object is linked in a list, which is 274 * hidden to caller. bpf_objects__<func> handlers deal with 275 * all objects. 276 */ 277 struct list_head list; 278 279 struct btf *btf; 280 struct btf_ext *btf_ext; 281 282 void *priv; 283 bpf_object_clear_priv_t clear_priv; 284 285 struct bpf_capabilities caps; 286 287 char path[]; 288 }; 289 #define obj_elf_valid(o) ((o)->efile.elf) 290 291 void bpf_program__unload(struct bpf_program *prog) 292 { 293 int i; 294 295 if (!prog) 296 return; 297 298 /* 299 * If the object is opened but the program was never loaded, 300 * it is possible that prog->instances.nr == -1. 301 */ 302 if (prog->instances.nr > 0) { 303 for (i = 0; i < prog->instances.nr; i++) 304 zclose(prog->instances.fds[i]); 305 } else if (prog->instances.nr != -1) { 306 pr_warning("Internal error: instances.nr is %d\n", 307 prog->instances.nr); 308 } 309 310 prog->instances.nr = -1; 311 zfree(&prog->instances.fds); 312 313 zclose(prog->btf_fd); 314 zfree(&prog->func_info); 315 zfree(&prog->line_info); 316 } 317 318 static void bpf_program__exit(struct bpf_program *prog) 319 { 320 if (!prog) 321 return; 322 323 if (prog->clear_priv) 324 prog->clear_priv(prog, prog->priv); 325 326 prog->priv = NULL; 327 prog->clear_priv = NULL; 328 329 bpf_program__unload(prog); 330 zfree(&prog->name); 331 zfree(&prog->section_name); 332 zfree(&prog->pin_name); 333 zfree(&prog->insns); 334 zfree(&prog->reloc_desc); 335 336 prog->nr_reloc = 0; 337 prog->insns_cnt = 0; 338 prog->idx = -1; 339 } 340 341 static char *__bpf_program__pin_name(struct bpf_program *prog) 342 { 343 char *name, *p; 344 345 name = p = strdup(prog->section_name); 346 while ((p = strchr(p, '/'))) 347 *p = '_'; 348 349 return name; 350 } 351 352 static int 353 bpf_program__init(void *data, size_t size, char *section_name, int idx, 354 struct bpf_program *prog) 355 { 356 const size_t bpf_insn_sz = sizeof(struct bpf_insn); 357 358 if (size == 0 || size % bpf_insn_sz) { 359 pr_warning("corrupted section '%s', size: %zu\n", 360 section_name, size); 361 return -EINVAL; 362 } 363 364 memset(prog, 0, sizeof(*prog)); 365 366 prog->section_name = strdup(section_name); 367 if (!prog->section_name) { 368 pr_warning("failed to alloc name for prog under section(%d) %s\n", 369 idx, section_name); 370 goto errout; 371 } 372 373 prog->pin_name = __bpf_program__pin_name(prog); 374 if (!prog->pin_name) { 375 pr_warning("failed to alloc pin name for prog under section(%d) %s\n", 376 idx, section_name); 377 goto errout; 378 } 379 380 prog->insns = malloc(size); 381 if (!prog->insns) { 382 pr_warning("failed to alloc insns for prog under section %s\n", 383 section_name); 384 goto errout; 385 } 386 prog->insns_cnt = size / bpf_insn_sz; 387 memcpy(prog->insns, data, size); 388 prog->idx = idx; 389 prog->instances.fds = NULL; 390 prog->instances.nr = -1; 391 prog->type = BPF_PROG_TYPE_UNSPEC; 392 prog->btf_fd = -1; 393 394 return 0; 395 errout: 396 bpf_program__exit(prog); 397 return -ENOMEM; 398 } 399 400 static int 401 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 402 char *section_name, int idx) 403 { 404 struct bpf_program prog, *progs; 405 int nr_progs, err; 406 407 err = bpf_program__init(data, size, section_name, idx, &prog); 408 if (err) 409 return err; 410 411 prog.caps = &obj->caps; 412 progs = obj->programs; 413 nr_progs = obj->nr_programs; 414 415 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 416 if (!progs) { 417 /* 418 * In this case the original obj->programs 419 * is still valid, so don't need special treat for 420 * bpf_close_object(). 421 */ 422 pr_warning("failed to alloc a new program under section '%s'\n", 423 section_name); 424 bpf_program__exit(&prog); 425 return -ENOMEM; 426 } 427 428 pr_debug("found program %s\n", prog.section_name); 429 obj->programs = progs; 430 obj->nr_programs = nr_progs + 1; 431 prog.obj = obj; 432 progs[nr_progs] = prog; 433 return 0; 434 } 435 436 static int 437 bpf_object__init_prog_names(struct bpf_object *obj) 438 { 439 Elf_Data *symbols = obj->efile.symbols; 440 struct bpf_program *prog; 441 size_t pi, si; 442 443 for (pi = 0; pi < obj->nr_programs; pi++) { 444 const char *name = NULL; 445 446 prog = &obj->programs[pi]; 447 448 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 449 si++) { 450 GElf_Sym sym; 451 452 if (!gelf_getsym(symbols, si, &sym)) 453 continue; 454 if (sym.st_shndx != prog->idx) 455 continue; 456 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 457 continue; 458 459 name = elf_strptr(obj->efile.elf, 460 obj->efile.strtabidx, 461 sym.st_name); 462 if (!name) { 463 pr_warning("failed to get sym name string for prog %s\n", 464 prog->section_name); 465 return -LIBBPF_ERRNO__LIBELF; 466 } 467 } 468 469 if (!name && prog->idx == obj->efile.text_shndx) 470 name = ".text"; 471 472 if (!name) { 473 pr_warning("failed to find sym for prog %s\n", 474 prog->section_name); 475 return -EINVAL; 476 } 477 478 prog->name = strdup(name); 479 if (!prog->name) { 480 pr_warning("failed to allocate memory for prog sym %s\n", 481 name); 482 return -ENOMEM; 483 } 484 } 485 486 return 0; 487 } 488 489 static struct bpf_object *bpf_object__new(const char *path, 490 void *obj_buf, 491 size_t obj_buf_sz) 492 { 493 struct bpf_object *obj; 494 char *end; 495 496 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 497 if (!obj) { 498 pr_warning("alloc memory failed for %s\n", path); 499 return ERR_PTR(-ENOMEM); 500 } 501 502 strcpy(obj->path, path); 503 /* Using basename() GNU version which doesn't modify arg. */ 504 strncpy(obj->name, basename((void *)path), sizeof(obj->name) - 1); 505 end = strchr(obj->name, '.'); 506 if (end) 507 *end = 0; 508 509 obj->efile.fd = -1; 510 /* 511 * Caller of this function should also call 512 * bpf_object__elf_finish() after data collection to return 513 * obj_buf to user. If not, we should duplicate the buffer to 514 * avoid user freeing them before elf finish. 515 */ 516 obj->efile.obj_buf = obj_buf; 517 obj->efile.obj_buf_sz = obj_buf_sz; 518 obj->efile.maps_shndx = -1; 519 obj->efile.btf_maps_shndx = -1; 520 obj->efile.data_shndx = -1; 521 obj->efile.rodata_shndx = -1; 522 obj->efile.bss_shndx = -1; 523 524 obj->loaded = false; 525 526 INIT_LIST_HEAD(&obj->list); 527 list_add(&obj->list, &bpf_objects_list); 528 return obj; 529 } 530 531 static void bpf_object__elf_finish(struct bpf_object *obj) 532 { 533 if (!obj_elf_valid(obj)) 534 return; 535 536 if (obj->efile.elf) { 537 elf_end(obj->efile.elf); 538 obj->efile.elf = NULL; 539 } 540 obj->efile.symbols = NULL; 541 obj->efile.data = NULL; 542 obj->efile.rodata = NULL; 543 obj->efile.bss = NULL; 544 545 zfree(&obj->efile.reloc); 546 obj->efile.nr_reloc = 0; 547 zclose(obj->efile.fd); 548 obj->efile.obj_buf = NULL; 549 obj->efile.obj_buf_sz = 0; 550 } 551 552 static int bpf_object__elf_init(struct bpf_object *obj) 553 { 554 int err = 0; 555 GElf_Ehdr *ep; 556 557 if (obj_elf_valid(obj)) { 558 pr_warning("elf init: internal error\n"); 559 return -LIBBPF_ERRNO__LIBELF; 560 } 561 562 if (obj->efile.obj_buf_sz > 0) { 563 /* 564 * obj_buf should have been validated by 565 * bpf_object__open_buffer(). 566 */ 567 obj->efile.elf = elf_memory(obj->efile.obj_buf, 568 obj->efile.obj_buf_sz); 569 } else { 570 obj->efile.fd = open(obj->path, O_RDONLY); 571 if (obj->efile.fd < 0) { 572 char errmsg[STRERR_BUFSIZE], *cp; 573 574 err = -errno; 575 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 576 pr_warning("failed to open %s: %s\n", obj->path, cp); 577 return err; 578 } 579 580 obj->efile.elf = elf_begin(obj->efile.fd, 581 LIBBPF_ELF_C_READ_MMAP, NULL); 582 } 583 584 if (!obj->efile.elf) { 585 pr_warning("failed to open %s as ELF file\n", obj->path); 586 err = -LIBBPF_ERRNO__LIBELF; 587 goto errout; 588 } 589 590 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 591 pr_warning("failed to get EHDR from %s\n", obj->path); 592 err = -LIBBPF_ERRNO__FORMAT; 593 goto errout; 594 } 595 ep = &obj->efile.ehdr; 596 597 /* Old LLVM set e_machine to EM_NONE */ 598 if (ep->e_type != ET_REL || 599 (ep->e_machine && ep->e_machine != EM_BPF)) { 600 pr_warning("%s is not an eBPF object file\n", obj->path); 601 err = -LIBBPF_ERRNO__FORMAT; 602 goto errout; 603 } 604 605 return 0; 606 errout: 607 bpf_object__elf_finish(obj); 608 return err; 609 } 610 611 static int bpf_object__check_endianness(struct bpf_object *obj) 612 { 613 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 614 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) 615 return 0; 616 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 617 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) 618 return 0; 619 #else 620 # error "Unrecognized __BYTE_ORDER__" 621 #endif 622 pr_warning("endianness mismatch.\n"); 623 return -LIBBPF_ERRNO__ENDIAN; 624 } 625 626 static int 627 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 628 { 629 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); 630 pr_debug("license of %s is %s\n", obj->path, obj->license); 631 return 0; 632 } 633 634 static int 635 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 636 { 637 __u32 kver; 638 639 if (size != sizeof(kver)) { 640 pr_warning("invalid kver section in %s\n", obj->path); 641 return -LIBBPF_ERRNO__FORMAT; 642 } 643 memcpy(&kver, data, sizeof(kver)); 644 obj->kern_version = kver; 645 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 646 return 0; 647 } 648 649 static int compare_bpf_map(const void *_a, const void *_b) 650 { 651 const struct bpf_map *a = _a; 652 const struct bpf_map *b = _b; 653 654 if (a->sec_idx != b->sec_idx) 655 return a->sec_idx - b->sec_idx; 656 return a->sec_offset - b->sec_offset; 657 } 658 659 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 660 { 661 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 662 type == BPF_MAP_TYPE_HASH_OF_MAPS) 663 return true; 664 return false; 665 } 666 667 static int bpf_object_search_section_size(const struct bpf_object *obj, 668 const char *name, size_t *d_size) 669 { 670 const GElf_Ehdr *ep = &obj->efile.ehdr; 671 Elf *elf = obj->efile.elf; 672 Elf_Scn *scn = NULL; 673 int idx = 0; 674 675 while ((scn = elf_nextscn(elf, scn)) != NULL) { 676 const char *sec_name; 677 Elf_Data *data; 678 GElf_Shdr sh; 679 680 idx++; 681 if (gelf_getshdr(scn, &sh) != &sh) { 682 pr_warning("failed to get section(%d) header from %s\n", 683 idx, obj->path); 684 return -EIO; 685 } 686 687 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 688 if (!sec_name) { 689 pr_warning("failed to get section(%d) name from %s\n", 690 idx, obj->path); 691 return -EIO; 692 } 693 694 if (strcmp(name, sec_name)) 695 continue; 696 697 data = elf_getdata(scn, 0); 698 if (!data) { 699 pr_warning("failed to get section(%d) data from %s(%s)\n", 700 idx, name, obj->path); 701 return -EIO; 702 } 703 704 *d_size = data->d_size; 705 return 0; 706 } 707 708 return -ENOENT; 709 } 710 711 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 712 __u32 *size) 713 { 714 int ret = -ENOENT; 715 size_t d_size; 716 717 *size = 0; 718 if (!name) { 719 return -EINVAL; 720 } else if (!strcmp(name, ".data")) { 721 if (obj->efile.data) 722 *size = obj->efile.data->d_size; 723 } else if (!strcmp(name, ".bss")) { 724 if (obj->efile.bss) 725 *size = obj->efile.bss->d_size; 726 } else if (!strcmp(name, ".rodata")) { 727 if (obj->efile.rodata) 728 *size = obj->efile.rodata->d_size; 729 } else { 730 ret = bpf_object_search_section_size(obj, name, &d_size); 731 if (!ret) 732 *size = d_size; 733 } 734 735 return *size ? 0 : ret; 736 } 737 738 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 739 __u32 *off) 740 { 741 Elf_Data *symbols = obj->efile.symbols; 742 const char *sname; 743 size_t si; 744 745 if (!name || !off) 746 return -EINVAL; 747 748 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 749 GElf_Sym sym; 750 751 if (!gelf_getsym(symbols, si, &sym)) 752 continue; 753 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 754 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 755 continue; 756 757 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 758 sym.st_name); 759 if (!sname) { 760 pr_warning("failed to get sym name string for var %s\n", 761 name); 762 return -EIO; 763 } 764 if (strcmp(name, sname) == 0) { 765 *off = sym.st_value; 766 return 0; 767 } 768 } 769 770 return -ENOENT; 771 } 772 773 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 774 { 775 struct bpf_map *new_maps; 776 size_t new_cap; 777 int i; 778 779 if (obj->nr_maps < obj->maps_cap) 780 return &obj->maps[obj->nr_maps++]; 781 782 new_cap = max((size_t)4, obj->maps_cap * 3 / 2); 783 new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps)); 784 if (!new_maps) { 785 pr_warning("alloc maps for object failed\n"); 786 return ERR_PTR(-ENOMEM); 787 } 788 789 obj->maps_cap = new_cap; 790 obj->maps = new_maps; 791 792 /* zero out new maps */ 793 memset(obj->maps + obj->nr_maps, 0, 794 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); 795 /* 796 * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) 797 * when failure (zclose won't close negative fd)). 798 */ 799 for (i = obj->nr_maps; i < obj->maps_cap; i++) { 800 obj->maps[i].fd = -1; 801 obj->maps[i].inner_map_fd = -1; 802 } 803 804 return &obj->maps[obj->nr_maps++]; 805 } 806 807 static int 808 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 809 int sec_idx, Elf_Data *data, void **data_buff) 810 { 811 char map_name[BPF_OBJ_NAME_LEN]; 812 struct bpf_map_def *def; 813 struct bpf_map *map; 814 815 map = bpf_object__add_map(obj); 816 if (IS_ERR(map)) 817 return PTR_ERR(map); 818 819 map->libbpf_type = type; 820 map->sec_idx = sec_idx; 821 map->sec_offset = 0; 822 snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, 823 libbpf_type_to_btf_name[type]); 824 map->name = strdup(map_name); 825 if (!map->name) { 826 pr_warning("failed to alloc map name\n"); 827 return -ENOMEM; 828 } 829 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n", 830 map_name, map->sec_idx, map->sec_offset); 831 832 def = &map->def; 833 def->type = BPF_MAP_TYPE_ARRAY; 834 def->key_size = sizeof(int); 835 def->value_size = data->d_size; 836 def->max_entries = 1; 837 def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0; 838 if (data_buff) { 839 *data_buff = malloc(data->d_size); 840 if (!*data_buff) { 841 zfree(&map->name); 842 pr_warning("failed to alloc map content buffer\n"); 843 return -ENOMEM; 844 } 845 memcpy(*data_buff, data->d_buf, data->d_size); 846 } 847 848 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 849 return 0; 850 } 851 852 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 853 { 854 int err; 855 856 if (!obj->caps.global_data) 857 return 0; 858 /* 859 * Populate obj->maps with libbpf internal maps. 860 */ 861 if (obj->efile.data_shndx >= 0) { 862 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 863 obj->efile.data_shndx, 864 obj->efile.data, 865 &obj->sections.data); 866 if (err) 867 return err; 868 } 869 if (obj->efile.rodata_shndx >= 0) { 870 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 871 obj->efile.rodata_shndx, 872 obj->efile.rodata, 873 &obj->sections.rodata); 874 if (err) 875 return err; 876 } 877 if (obj->efile.bss_shndx >= 0) { 878 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 879 obj->efile.bss_shndx, 880 obj->efile.bss, NULL); 881 if (err) 882 return err; 883 } 884 return 0; 885 } 886 887 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) 888 { 889 Elf_Data *symbols = obj->efile.symbols; 890 int i, map_def_sz = 0, nr_maps = 0, nr_syms; 891 Elf_Data *data = NULL; 892 Elf_Scn *scn; 893 894 if (obj->efile.maps_shndx < 0) 895 return 0; 896 897 if (!symbols) 898 return -EINVAL; 899 900 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); 901 if (scn) 902 data = elf_getdata(scn, NULL); 903 if (!scn || !data) { 904 pr_warning("failed to get Elf_Data from map section %d\n", 905 obj->efile.maps_shndx); 906 return -EINVAL; 907 } 908 909 /* 910 * Count number of maps. Each map has a name. 911 * Array of maps is not supported: only the first element is 912 * considered. 913 * 914 * TODO: Detect array of map and report error. 915 */ 916 nr_syms = symbols->d_size / sizeof(GElf_Sym); 917 for (i = 0; i < nr_syms; i++) { 918 GElf_Sym sym; 919 920 if (!gelf_getsym(symbols, i, &sym)) 921 continue; 922 if (sym.st_shndx != obj->efile.maps_shndx) 923 continue; 924 nr_maps++; 925 } 926 /* Assume equally sized map definitions */ 927 pr_debug("maps in %s: %d maps in %zd bytes\n", 928 obj->path, nr_maps, data->d_size); 929 930 map_def_sz = data->d_size / nr_maps; 931 if (!data->d_size || (data->d_size % nr_maps) != 0) { 932 pr_warning("unable to determine map definition size " 933 "section %s, %d maps in %zd bytes\n", 934 obj->path, nr_maps, data->d_size); 935 return -EINVAL; 936 } 937 938 /* Fill obj->maps using data in "maps" section. */ 939 for (i = 0; i < nr_syms; i++) { 940 GElf_Sym sym; 941 const char *map_name; 942 struct bpf_map_def *def; 943 struct bpf_map *map; 944 945 if (!gelf_getsym(symbols, i, &sym)) 946 continue; 947 if (sym.st_shndx != obj->efile.maps_shndx) 948 continue; 949 950 map = bpf_object__add_map(obj); 951 if (IS_ERR(map)) 952 return PTR_ERR(map); 953 954 map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 955 sym.st_name); 956 if (!map_name) { 957 pr_warning("failed to get map #%d name sym string for obj %s\n", 958 i, obj->path); 959 return -LIBBPF_ERRNO__FORMAT; 960 } 961 962 map->libbpf_type = LIBBPF_MAP_UNSPEC; 963 map->sec_idx = sym.st_shndx; 964 map->sec_offset = sym.st_value; 965 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", 966 map_name, map->sec_idx, map->sec_offset); 967 if (sym.st_value + map_def_sz > data->d_size) { 968 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 969 obj->path, map_name); 970 return -EINVAL; 971 } 972 973 map->name = strdup(map_name); 974 if (!map->name) { 975 pr_warning("failed to alloc map name\n"); 976 return -ENOMEM; 977 } 978 pr_debug("map %d is \"%s\"\n", i, map->name); 979 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 980 /* 981 * If the definition of the map in the object file fits in 982 * bpf_map_def, copy it. Any extra fields in our version 983 * of bpf_map_def will default to zero as a result of the 984 * calloc above. 985 */ 986 if (map_def_sz <= sizeof(struct bpf_map_def)) { 987 memcpy(&map->def, def, map_def_sz); 988 } else { 989 /* 990 * Here the map structure being read is bigger than what 991 * we expect, truncate if the excess bits are all zero. 992 * If they are not zero, reject this map as 993 * incompatible. 994 */ 995 char *b; 996 for (b = ((char *)def) + sizeof(struct bpf_map_def); 997 b < ((char *)def) + map_def_sz; b++) { 998 if (*b != 0) { 999 pr_warning("maps section in %s: \"%s\" " 1000 "has unrecognized, non-zero " 1001 "options\n", 1002 obj->path, map_name); 1003 if (strict) 1004 return -EINVAL; 1005 } 1006 } 1007 memcpy(&map->def, def, sizeof(struct bpf_map_def)); 1008 } 1009 } 1010 return 0; 1011 } 1012 1013 static const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, 1014 __u32 id) 1015 { 1016 const struct btf_type *t = btf__type_by_id(btf, id); 1017 1018 while (true) { 1019 switch (BTF_INFO_KIND(t->info)) { 1020 case BTF_KIND_VOLATILE: 1021 case BTF_KIND_CONST: 1022 case BTF_KIND_RESTRICT: 1023 case BTF_KIND_TYPEDEF: 1024 t = btf__type_by_id(btf, t->type); 1025 break; 1026 default: 1027 return t; 1028 } 1029 } 1030 } 1031 1032 /* 1033 * Fetch integer attribute of BTF map definition. Such attributes are 1034 * represented using a pointer to an array, in which dimensionality of array 1035 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 1036 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 1037 * type definition, while using only sizeof(void *) space in ELF data section. 1038 */ 1039 static bool get_map_field_int(const char *map_name, const struct btf *btf, 1040 const struct btf_type *def, 1041 const struct btf_member *m, __u32 *res) { 1042 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type); 1043 const char *name = btf__name_by_offset(btf, m->name_off); 1044 const struct btf_array *arr_info; 1045 const struct btf_type *arr_t; 1046 1047 if (BTF_INFO_KIND(t->info) != BTF_KIND_PTR) { 1048 pr_warning("map '%s': attr '%s': expected PTR, got %u.\n", 1049 map_name, name, BTF_INFO_KIND(t->info)); 1050 return false; 1051 } 1052 1053 arr_t = btf__type_by_id(btf, t->type); 1054 if (!arr_t) { 1055 pr_warning("map '%s': attr '%s': type [%u] not found.\n", 1056 map_name, name, t->type); 1057 return false; 1058 } 1059 if (BTF_INFO_KIND(arr_t->info) != BTF_KIND_ARRAY) { 1060 pr_warning("map '%s': attr '%s': expected ARRAY, got %u.\n", 1061 map_name, name, BTF_INFO_KIND(arr_t->info)); 1062 return false; 1063 } 1064 arr_info = (const void *)(arr_t + 1); 1065 *res = arr_info->nelems; 1066 return true; 1067 } 1068 1069 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 1070 const struct btf_type *sec, 1071 int var_idx, int sec_idx, 1072 const Elf_Data *data, bool strict) 1073 { 1074 const struct btf_type *var, *def, *t; 1075 const struct btf_var_secinfo *vi; 1076 const struct btf_var *var_extra; 1077 const struct btf_member *m; 1078 const char *map_name; 1079 struct bpf_map *map; 1080 int vlen, i; 1081 1082 vi = (const struct btf_var_secinfo *)(const void *)(sec + 1) + var_idx; 1083 var = btf__type_by_id(obj->btf, vi->type); 1084 var_extra = (const void *)(var + 1); 1085 map_name = btf__name_by_offset(obj->btf, var->name_off); 1086 vlen = BTF_INFO_VLEN(var->info); 1087 1088 if (map_name == NULL || map_name[0] == '\0') { 1089 pr_warning("map #%d: empty name.\n", var_idx); 1090 return -EINVAL; 1091 } 1092 if ((__u64)vi->offset + vi->size > data->d_size) { 1093 pr_warning("map '%s' BTF data is corrupted.\n", map_name); 1094 return -EINVAL; 1095 } 1096 if (BTF_INFO_KIND(var->info) != BTF_KIND_VAR) { 1097 pr_warning("map '%s': unexpected var kind %u.\n", 1098 map_name, BTF_INFO_KIND(var->info)); 1099 return -EINVAL; 1100 } 1101 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && 1102 var_extra->linkage != BTF_VAR_STATIC) { 1103 pr_warning("map '%s': unsupported var linkage %u.\n", 1104 map_name, var_extra->linkage); 1105 return -EOPNOTSUPP; 1106 } 1107 1108 def = skip_mods_and_typedefs(obj->btf, var->type); 1109 if (BTF_INFO_KIND(def->info) != BTF_KIND_STRUCT) { 1110 pr_warning("map '%s': unexpected def kind %u.\n", 1111 map_name, BTF_INFO_KIND(var->info)); 1112 return -EINVAL; 1113 } 1114 if (def->size > vi->size) { 1115 pr_warning("map '%s': invalid def size.\n", map_name); 1116 return -EINVAL; 1117 } 1118 1119 map = bpf_object__add_map(obj); 1120 if (IS_ERR(map)) 1121 return PTR_ERR(map); 1122 map->name = strdup(map_name); 1123 if (!map->name) { 1124 pr_warning("map '%s': failed to alloc map name.\n", map_name); 1125 return -ENOMEM; 1126 } 1127 map->libbpf_type = LIBBPF_MAP_UNSPEC; 1128 map->def.type = BPF_MAP_TYPE_UNSPEC; 1129 map->sec_idx = sec_idx; 1130 map->sec_offset = vi->offset; 1131 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 1132 map_name, map->sec_idx, map->sec_offset); 1133 1134 vlen = BTF_INFO_VLEN(def->info); 1135 m = (const void *)(def + 1); 1136 for (i = 0; i < vlen; i++, m++) { 1137 const char *name = btf__name_by_offset(obj->btf, m->name_off); 1138 1139 if (!name) { 1140 pr_warning("map '%s': invalid field #%d.\n", 1141 map_name, i); 1142 return -EINVAL; 1143 } 1144 if (strcmp(name, "type") == 0) { 1145 if (!get_map_field_int(map_name, obj->btf, def, m, 1146 &map->def.type)) 1147 return -EINVAL; 1148 pr_debug("map '%s': found type = %u.\n", 1149 map_name, map->def.type); 1150 } else if (strcmp(name, "max_entries") == 0) { 1151 if (!get_map_field_int(map_name, obj->btf, def, m, 1152 &map->def.max_entries)) 1153 return -EINVAL; 1154 pr_debug("map '%s': found max_entries = %u.\n", 1155 map_name, map->def.max_entries); 1156 } else if (strcmp(name, "map_flags") == 0) { 1157 if (!get_map_field_int(map_name, obj->btf, def, m, 1158 &map->def.map_flags)) 1159 return -EINVAL; 1160 pr_debug("map '%s': found map_flags = %u.\n", 1161 map_name, map->def.map_flags); 1162 } else if (strcmp(name, "key_size") == 0) { 1163 __u32 sz; 1164 1165 if (!get_map_field_int(map_name, obj->btf, def, m, 1166 &sz)) 1167 return -EINVAL; 1168 pr_debug("map '%s': found key_size = %u.\n", 1169 map_name, sz); 1170 if (map->def.key_size && map->def.key_size != sz) { 1171 pr_warning("map '%s': conflicting key size %u != %u.\n", 1172 map_name, map->def.key_size, sz); 1173 return -EINVAL; 1174 } 1175 map->def.key_size = sz; 1176 } else if (strcmp(name, "key") == 0) { 1177 __s64 sz; 1178 1179 t = btf__type_by_id(obj->btf, m->type); 1180 if (!t) { 1181 pr_warning("map '%s': key type [%d] not found.\n", 1182 map_name, m->type); 1183 return -EINVAL; 1184 } 1185 if (BTF_INFO_KIND(t->info) != BTF_KIND_PTR) { 1186 pr_warning("map '%s': key spec is not PTR: %u.\n", 1187 map_name, BTF_INFO_KIND(t->info)); 1188 return -EINVAL; 1189 } 1190 sz = btf__resolve_size(obj->btf, t->type); 1191 if (sz < 0) { 1192 pr_warning("map '%s': can't determine key size for type [%u]: %lld.\n", 1193 map_name, t->type, sz); 1194 return sz; 1195 } 1196 pr_debug("map '%s': found key [%u], sz = %lld.\n", 1197 map_name, t->type, sz); 1198 if (map->def.key_size && map->def.key_size != sz) { 1199 pr_warning("map '%s': conflicting key size %u != %lld.\n", 1200 map_name, map->def.key_size, sz); 1201 return -EINVAL; 1202 } 1203 map->def.key_size = sz; 1204 map->btf_key_type_id = t->type; 1205 } else if (strcmp(name, "value_size") == 0) { 1206 __u32 sz; 1207 1208 if (!get_map_field_int(map_name, obj->btf, def, m, 1209 &sz)) 1210 return -EINVAL; 1211 pr_debug("map '%s': found value_size = %u.\n", 1212 map_name, sz); 1213 if (map->def.value_size && map->def.value_size != sz) { 1214 pr_warning("map '%s': conflicting value size %u != %u.\n", 1215 map_name, map->def.value_size, sz); 1216 return -EINVAL; 1217 } 1218 map->def.value_size = sz; 1219 } else if (strcmp(name, "value") == 0) { 1220 __s64 sz; 1221 1222 t = btf__type_by_id(obj->btf, m->type); 1223 if (!t) { 1224 pr_warning("map '%s': value type [%d] not found.\n", 1225 map_name, m->type); 1226 return -EINVAL; 1227 } 1228 if (BTF_INFO_KIND(t->info) != BTF_KIND_PTR) { 1229 pr_warning("map '%s': value spec is not PTR: %u.\n", 1230 map_name, BTF_INFO_KIND(t->info)); 1231 return -EINVAL; 1232 } 1233 sz = btf__resolve_size(obj->btf, t->type); 1234 if (sz < 0) { 1235 pr_warning("map '%s': can't determine value size for type [%u]: %lld.\n", 1236 map_name, t->type, sz); 1237 return sz; 1238 } 1239 pr_debug("map '%s': found value [%u], sz = %lld.\n", 1240 map_name, t->type, sz); 1241 if (map->def.value_size && map->def.value_size != sz) { 1242 pr_warning("map '%s': conflicting value size %u != %lld.\n", 1243 map_name, map->def.value_size, sz); 1244 return -EINVAL; 1245 } 1246 map->def.value_size = sz; 1247 map->btf_value_type_id = t->type; 1248 } else { 1249 if (strict) { 1250 pr_warning("map '%s': unknown field '%s'.\n", 1251 map_name, name); 1252 return -ENOTSUP; 1253 } 1254 pr_debug("map '%s': ignoring unknown field '%s'.\n", 1255 map_name, name); 1256 } 1257 } 1258 1259 if (map->def.type == BPF_MAP_TYPE_UNSPEC) { 1260 pr_warning("map '%s': map type isn't specified.\n", map_name); 1261 return -EINVAL; 1262 } 1263 1264 return 0; 1265 } 1266 1267 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) 1268 { 1269 const struct btf_type *sec = NULL; 1270 int nr_types, i, vlen, err; 1271 const struct btf_type *t; 1272 const char *name; 1273 Elf_Data *data; 1274 Elf_Scn *scn; 1275 1276 if (obj->efile.btf_maps_shndx < 0) 1277 return 0; 1278 1279 scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx); 1280 if (scn) 1281 data = elf_getdata(scn, NULL); 1282 if (!scn || !data) { 1283 pr_warning("failed to get Elf_Data from map section %d (%s)\n", 1284 obj->efile.maps_shndx, MAPS_ELF_SEC); 1285 return -EINVAL; 1286 } 1287 1288 nr_types = btf__get_nr_types(obj->btf); 1289 for (i = 1; i <= nr_types; i++) { 1290 t = btf__type_by_id(obj->btf, i); 1291 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) 1292 continue; 1293 name = btf__name_by_offset(obj->btf, t->name_off); 1294 if (strcmp(name, MAPS_ELF_SEC) == 0) { 1295 sec = t; 1296 break; 1297 } 1298 } 1299 1300 if (!sec) { 1301 pr_warning("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 1302 return -ENOENT; 1303 } 1304 1305 vlen = BTF_INFO_VLEN(sec->info); 1306 for (i = 0; i < vlen; i++) { 1307 err = bpf_object__init_user_btf_map(obj, sec, i, 1308 obj->efile.btf_maps_shndx, 1309 data, strict); 1310 if (err) 1311 return err; 1312 } 1313 1314 return 0; 1315 } 1316 1317 static int bpf_object__init_maps(struct bpf_object *obj, int flags) 1318 { 1319 bool strict = !(flags & MAPS_RELAX_COMPAT); 1320 int err; 1321 1322 err = bpf_object__init_user_maps(obj, strict); 1323 if (err) 1324 return err; 1325 1326 err = bpf_object__init_user_btf_maps(obj, strict); 1327 if (err) 1328 return err; 1329 1330 err = bpf_object__init_global_data_maps(obj); 1331 if (err) 1332 return err; 1333 1334 if (obj->nr_maps) { 1335 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), 1336 compare_bpf_map); 1337 } 1338 return 0; 1339 } 1340 1341 static bool section_have_execinstr(struct bpf_object *obj, int idx) 1342 { 1343 Elf_Scn *scn; 1344 GElf_Shdr sh; 1345 1346 scn = elf_getscn(obj->efile.elf, idx); 1347 if (!scn) 1348 return false; 1349 1350 if (gelf_getshdr(scn, &sh) != &sh) 1351 return false; 1352 1353 if (sh.sh_flags & SHF_EXECINSTR) 1354 return true; 1355 1356 return false; 1357 } 1358 1359 static void bpf_object__sanitize_btf(struct bpf_object *obj) 1360 { 1361 bool has_datasec = obj->caps.btf_datasec; 1362 bool has_func = obj->caps.btf_func; 1363 struct btf *btf = obj->btf; 1364 struct btf_type *t; 1365 int i, j, vlen; 1366 __u16 kind; 1367 1368 if (!obj->btf || (has_func && has_datasec)) 1369 return; 1370 1371 for (i = 1; i <= btf__get_nr_types(btf); i++) { 1372 t = (struct btf_type *)btf__type_by_id(btf, i); 1373 kind = BTF_INFO_KIND(t->info); 1374 1375 if (!has_datasec && kind == BTF_KIND_VAR) { 1376 /* replace VAR with INT */ 1377 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 1378 t->size = sizeof(int); 1379 *(int *)(t+1) = BTF_INT_ENC(0, 0, 32); 1380 } else if (!has_datasec && kind == BTF_KIND_DATASEC) { 1381 /* replace DATASEC with STRUCT */ 1382 struct btf_var_secinfo *v = (void *)(t + 1); 1383 struct btf_member *m = (void *)(t + 1); 1384 struct btf_type *vt; 1385 char *name; 1386 1387 name = (char *)btf__name_by_offset(btf, t->name_off); 1388 while (*name) { 1389 if (*name == '.') 1390 *name = '_'; 1391 name++; 1392 } 1393 1394 vlen = BTF_INFO_VLEN(t->info); 1395 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 1396 for (j = 0; j < vlen; j++, v++, m++) { 1397 /* order of field assignments is important */ 1398 m->offset = v->offset * 8; 1399 m->type = v->type; 1400 /* preserve variable name as member name */ 1401 vt = (void *)btf__type_by_id(btf, v->type); 1402 m->name_off = vt->name_off; 1403 } 1404 } else if (!has_func && kind == BTF_KIND_FUNC_PROTO) { 1405 /* replace FUNC_PROTO with ENUM */ 1406 vlen = BTF_INFO_VLEN(t->info); 1407 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 1408 t->size = sizeof(__u32); /* kernel enforced */ 1409 } else if (!has_func && kind == BTF_KIND_FUNC) { 1410 /* replace FUNC with TYPEDEF */ 1411 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 1412 } 1413 } 1414 } 1415 1416 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) 1417 { 1418 if (!obj->btf_ext) 1419 return; 1420 1421 if (!obj->caps.btf_func) { 1422 btf_ext__free(obj->btf_ext); 1423 obj->btf_ext = NULL; 1424 } 1425 } 1426 1427 static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj) 1428 { 1429 return obj->efile.btf_maps_shndx >= 0; 1430 } 1431 1432 static int bpf_object__init_btf(struct bpf_object *obj, 1433 Elf_Data *btf_data, 1434 Elf_Data *btf_ext_data) 1435 { 1436 bool btf_required = bpf_object__is_btf_mandatory(obj); 1437 int err = 0; 1438 1439 if (btf_data) { 1440 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 1441 if (IS_ERR(obj->btf)) { 1442 pr_warning("Error loading ELF section %s: %d.\n", 1443 BTF_ELF_SEC, err); 1444 goto out; 1445 } 1446 err = btf__finalize_data(obj, obj->btf); 1447 if (err) { 1448 pr_warning("Error finalizing %s: %d.\n", 1449 BTF_ELF_SEC, err); 1450 goto out; 1451 } 1452 } 1453 if (btf_ext_data) { 1454 if (!obj->btf) { 1455 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 1456 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 1457 goto out; 1458 } 1459 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 1460 btf_ext_data->d_size); 1461 if (IS_ERR(obj->btf_ext)) { 1462 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1463 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); 1464 obj->btf_ext = NULL; 1465 goto out; 1466 } 1467 } 1468 out: 1469 if (err || IS_ERR(obj->btf)) { 1470 if (btf_required) 1471 err = err ? : PTR_ERR(obj->btf); 1472 else 1473 err = 0; 1474 if (!IS_ERR_OR_NULL(obj->btf)) 1475 btf__free(obj->btf); 1476 obj->btf = NULL; 1477 } 1478 if (btf_required && !obj->btf) { 1479 pr_warning("BTF is required, but is missing or corrupted.\n"); 1480 return err == 0 ? -ENOENT : err; 1481 } 1482 return 0; 1483 } 1484 1485 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 1486 { 1487 int err = 0; 1488 1489 if (!obj->btf) 1490 return 0; 1491 1492 bpf_object__sanitize_btf(obj); 1493 bpf_object__sanitize_btf_ext(obj); 1494 1495 err = btf__load(obj->btf); 1496 if (err) { 1497 pr_warning("Error loading %s into kernel: %d.\n", 1498 BTF_ELF_SEC, err); 1499 btf__free(obj->btf); 1500 obj->btf = NULL; 1501 if (bpf_object__is_btf_mandatory(obj)) 1502 return err; 1503 } 1504 return 0; 1505 } 1506 1507 static int bpf_object__elf_collect(struct bpf_object *obj, int flags) 1508 { 1509 Elf *elf = obj->efile.elf; 1510 GElf_Ehdr *ep = &obj->efile.ehdr; 1511 Elf_Data *btf_ext_data = NULL; 1512 Elf_Data *btf_data = NULL; 1513 Elf_Scn *scn = NULL; 1514 int idx = 0, err = 0; 1515 1516 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1517 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 1518 pr_warning("failed to get e_shstrndx from %s\n", obj->path); 1519 return -LIBBPF_ERRNO__FORMAT; 1520 } 1521 1522 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1523 char *name; 1524 GElf_Shdr sh; 1525 Elf_Data *data; 1526 1527 idx++; 1528 if (gelf_getshdr(scn, &sh) != &sh) { 1529 pr_warning("failed to get section(%d) header from %s\n", 1530 idx, obj->path); 1531 return -LIBBPF_ERRNO__FORMAT; 1532 } 1533 1534 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 1535 if (!name) { 1536 pr_warning("failed to get section(%d) name from %s\n", 1537 idx, obj->path); 1538 return -LIBBPF_ERRNO__FORMAT; 1539 } 1540 1541 data = elf_getdata(scn, 0); 1542 if (!data) { 1543 pr_warning("failed to get section(%d) data from %s(%s)\n", 1544 idx, name, obj->path); 1545 return -LIBBPF_ERRNO__FORMAT; 1546 } 1547 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 1548 idx, name, (unsigned long)data->d_size, 1549 (int)sh.sh_link, (unsigned long)sh.sh_flags, 1550 (int)sh.sh_type); 1551 1552 if (strcmp(name, "license") == 0) { 1553 err = bpf_object__init_license(obj, 1554 data->d_buf, 1555 data->d_size); 1556 if (err) 1557 return err; 1558 } else if (strcmp(name, "version") == 0) { 1559 err = bpf_object__init_kversion(obj, 1560 data->d_buf, 1561 data->d_size); 1562 if (err) 1563 return err; 1564 } else if (strcmp(name, "maps") == 0) { 1565 obj->efile.maps_shndx = idx; 1566 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 1567 obj->efile.btf_maps_shndx = idx; 1568 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 1569 btf_data = data; 1570 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 1571 btf_ext_data = data; 1572 } else if (sh.sh_type == SHT_SYMTAB) { 1573 if (obj->efile.symbols) { 1574 pr_warning("bpf: multiple SYMTAB in %s\n", 1575 obj->path); 1576 return -LIBBPF_ERRNO__FORMAT; 1577 } 1578 obj->efile.symbols = data; 1579 obj->efile.strtabidx = sh.sh_link; 1580 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 1581 if (sh.sh_flags & SHF_EXECINSTR) { 1582 if (strcmp(name, ".text") == 0) 1583 obj->efile.text_shndx = idx; 1584 err = bpf_object__add_program(obj, data->d_buf, 1585 data->d_size, name, idx); 1586 if (err) { 1587 char errmsg[STRERR_BUFSIZE]; 1588 char *cp = libbpf_strerror_r(-err, errmsg, 1589 sizeof(errmsg)); 1590 1591 pr_warning("failed to alloc program %s (%s): %s", 1592 name, obj->path, cp); 1593 return err; 1594 } 1595 } else if (strcmp(name, ".data") == 0) { 1596 obj->efile.data = data; 1597 obj->efile.data_shndx = idx; 1598 } else if (strcmp(name, ".rodata") == 0) { 1599 obj->efile.rodata = data; 1600 obj->efile.rodata_shndx = idx; 1601 } else { 1602 pr_debug("skip section(%d) %s\n", idx, name); 1603 } 1604 } else if (sh.sh_type == SHT_REL) { 1605 int nr_reloc = obj->efile.nr_reloc; 1606 void *reloc = obj->efile.reloc; 1607 int sec = sh.sh_info; /* points to other section */ 1608 1609 /* Only do relo for section with exec instructions */ 1610 if (!section_have_execinstr(obj, sec)) { 1611 pr_debug("skip relo %s(%d) for section(%d)\n", 1612 name, idx, sec); 1613 continue; 1614 } 1615 1616 reloc = reallocarray(reloc, nr_reloc + 1, 1617 sizeof(*obj->efile.reloc)); 1618 if (!reloc) { 1619 pr_warning("realloc failed\n"); 1620 return -ENOMEM; 1621 } 1622 1623 obj->efile.reloc = reloc; 1624 obj->efile.nr_reloc++; 1625 1626 obj->efile.reloc[nr_reloc].shdr = sh; 1627 obj->efile.reloc[nr_reloc].data = data; 1628 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { 1629 obj->efile.bss = data; 1630 obj->efile.bss_shndx = idx; 1631 } else { 1632 pr_debug("skip section(%d) %s\n", idx, name); 1633 } 1634 } 1635 1636 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 1637 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 1638 return -LIBBPF_ERRNO__FORMAT; 1639 } 1640 err = bpf_object__init_btf(obj, btf_data, btf_ext_data); 1641 if (!err) 1642 err = bpf_object__init_maps(obj, flags); 1643 if (!err) 1644 err = bpf_object__sanitize_and_load_btf(obj); 1645 if (!err) 1646 err = bpf_object__init_prog_names(obj); 1647 return err; 1648 } 1649 1650 static struct bpf_program * 1651 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 1652 { 1653 struct bpf_program *prog; 1654 size_t i; 1655 1656 for (i = 0; i < obj->nr_programs; i++) { 1657 prog = &obj->programs[i]; 1658 if (prog->idx == idx) 1659 return prog; 1660 } 1661 return NULL; 1662 } 1663 1664 struct bpf_program * 1665 bpf_object__find_program_by_title(const struct bpf_object *obj, 1666 const char *title) 1667 { 1668 struct bpf_program *pos; 1669 1670 bpf_object__for_each_program(pos, obj) { 1671 if (pos->section_name && !strcmp(pos->section_name, title)) 1672 return pos; 1673 } 1674 return NULL; 1675 } 1676 1677 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 1678 int shndx) 1679 { 1680 return shndx == obj->efile.data_shndx || 1681 shndx == obj->efile.bss_shndx || 1682 shndx == obj->efile.rodata_shndx; 1683 } 1684 1685 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 1686 int shndx) 1687 { 1688 return shndx == obj->efile.maps_shndx || 1689 shndx == obj->efile.btf_maps_shndx; 1690 } 1691 1692 static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, 1693 int shndx) 1694 { 1695 return shndx == obj->efile.text_shndx || 1696 bpf_object__shndx_is_maps(obj, shndx) || 1697 bpf_object__shndx_is_data(obj, shndx); 1698 } 1699 1700 static enum libbpf_map_type 1701 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 1702 { 1703 if (shndx == obj->efile.data_shndx) 1704 return LIBBPF_MAP_DATA; 1705 else if (shndx == obj->efile.bss_shndx) 1706 return LIBBPF_MAP_BSS; 1707 else if (shndx == obj->efile.rodata_shndx) 1708 return LIBBPF_MAP_RODATA; 1709 else 1710 return LIBBPF_MAP_UNSPEC; 1711 } 1712 1713 static int 1714 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 1715 Elf_Data *data, struct bpf_object *obj) 1716 { 1717 Elf_Data *symbols = obj->efile.symbols; 1718 struct bpf_map *maps = obj->maps; 1719 size_t nr_maps = obj->nr_maps; 1720 int i, nrels; 1721 1722 pr_debug("collecting relocating info for: '%s'\n", prog->section_name); 1723 nrels = shdr->sh_size / shdr->sh_entsize; 1724 1725 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 1726 if (!prog->reloc_desc) { 1727 pr_warning("failed to alloc memory in relocation\n"); 1728 return -ENOMEM; 1729 } 1730 prog->nr_reloc = nrels; 1731 1732 for (i = 0; i < nrels; i++) { 1733 struct bpf_insn *insns = prog->insns; 1734 enum libbpf_map_type type; 1735 unsigned int insn_idx; 1736 unsigned int shdr_idx; 1737 const char *name; 1738 size_t map_idx; 1739 GElf_Sym sym; 1740 GElf_Rel rel; 1741 1742 if (!gelf_getrel(data, i, &rel)) { 1743 pr_warning("relocation: failed to get %d reloc\n", i); 1744 return -LIBBPF_ERRNO__FORMAT; 1745 } 1746 1747 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 1748 pr_warning("relocation: symbol %"PRIx64" not found\n", 1749 GELF_R_SYM(rel.r_info)); 1750 return -LIBBPF_ERRNO__FORMAT; 1751 } 1752 1753 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1754 sym.st_name) ? : "<?>"; 1755 1756 pr_debug("relo for %lld value %lld name %d (\'%s\')\n", 1757 (long long) (rel.r_info >> 32), 1758 (long long) sym.st_value, sym.st_name, name); 1759 1760 shdr_idx = sym.st_shndx; 1761 if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { 1762 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", 1763 prog->section_name, shdr_idx); 1764 return -LIBBPF_ERRNO__RELOC; 1765 } 1766 1767 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 1768 pr_debug("relocation: insn_idx=%u\n", insn_idx); 1769 1770 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 1771 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 1772 pr_warning("incorrect bpf_call opcode\n"); 1773 return -LIBBPF_ERRNO__RELOC; 1774 } 1775 prog->reloc_desc[i].type = RELO_CALL; 1776 prog->reloc_desc[i].insn_idx = insn_idx; 1777 prog->reloc_desc[i].text_off = sym.st_value; 1778 obj->has_pseudo_calls = true; 1779 continue; 1780 } 1781 1782 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 1783 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 1784 insn_idx, insns[insn_idx].code); 1785 return -LIBBPF_ERRNO__RELOC; 1786 } 1787 1788 if (bpf_object__shndx_is_maps(obj, shdr_idx) || 1789 bpf_object__shndx_is_data(obj, shdr_idx)) { 1790 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 1791 if (type != LIBBPF_MAP_UNSPEC) { 1792 if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { 1793 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", 1794 name, insn_idx, insns[insn_idx].code); 1795 return -LIBBPF_ERRNO__RELOC; 1796 } 1797 if (!obj->caps.global_data) { 1798 pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", 1799 name, insn_idx); 1800 return -LIBBPF_ERRNO__RELOC; 1801 } 1802 } 1803 1804 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 1805 if (maps[map_idx].libbpf_type != type) 1806 continue; 1807 if (type != LIBBPF_MAP_UNSPEC || 1808 (maps[map_idx].sec_idx == sym.st_shndx && 1809 maps[map_idx].sec_offset == sym.st_value)) { 1810 pr_debug("relocation: found map %zd (%s, sec_idx %d, offset %zu) for insn %u\n", 1811 map_idx, maps[map_idx].name, 1812 maps[map_idx].sec_idx, 1813 maps[map_idx].sec_offset, 1814 insn_idx); 1815 break; 1816 } 1817 } 1818 1819 if (map_idx >= nr_maps) { 1820 pr_warning("bpf relocation: map_idx %d larger than %d\n", 1821 (int)map_idx, (int)nr_maps - 1); 1822 return -LIBBPF_ERRNO__RELOC; 1823 } 1824 1825 prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? 1826 RELO_DATA : RELO_LD64; 1827 prog->reloc_desc[i].insn_idx = insn_idx; 1828 prog->reloc_desc[i].map_idx = map_idx; 1829 } 1830 } 1831 return 0; 1832 } 1833 1834 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) 1835 { 1836 struct bpf_map_def *def = &map->def; 1837 __u32 key_type_id = 0, value_type_id = 0; 1838 int ret; 1839 1840 /* if it's BTF-defined map, we don't need to search for type IDs */ 1841 if (map->sec_idx == obj->efile.btf_maps_shndx) 1842 return 0; 1843 1844 if (!bpf_map__is_internal(map)) { 1845 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, 1846 def->value_size, &key_type_id, 1847 &value_type_id); 1848 } else { 1849 /* 1850 * LLVM annotates global data differently in BTF, that is, 1851 * only as '.data', '.bss' or '.rodata'. 1852 */ 1853 ret = btf__find_by_name(obj->btf, 1854 libbpf_type_to_btf_name[map->libbpf_type]); 1855 } 1856 if (ret < 0) 1857 return ret; 1858 1859 map->btf_key_type_id = key_type_id; 1860 map->btf_value_type_id = bpf_map__is_internal(map) ? 1861 ret : value_type_id; 1862 return 0; 1863 } 1864 1865 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 1866 { 1867 struct bpf_map_info info = {}; 1868 __u32 len = sizeof(info); 1869 int new_fd, err; 1870 char *new_name; 1871 1872 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1873 if (err) 1874 return err; 1875 1876 new_name = strdup(info.name); 1877 if (!new_name) 1878 return -errno; 1879 1880 new_fd = open("/", O_RDONLY | O_CLOEXEC); 1881 if (new_fd < 0) 1882 goto err_free_new_name; 1883 1884 new_fd = dup3(fd, new_fd, O_CLOEXEC); 1885 if (new_fd < 0) 1886 goto err_close_new_fd; 1887 1888 err = zclose(map->fd); 1889 if (err) 1890 goto err_close_new_fd; 1891 free(map->name); 1892 1893 map->fd = new_fd; 1894 map->name = new_name; 1895 map->def.type = info.type; 1896 map->def.key_size = info.key_size; 1897 map->def.value_size = info.value_size; 1898 map->def.max_entries = info.max_entries; 1899 map->def.map_flags = info.map_flags; 1900 map->btf_key_type_id = info.btf_key_type_id; 1901 map->btf_value_type_id = info.btf_value_type_id; 1902 1903 return 0; 1904 1905 err_close_new_fd: 1906 close(new_fd); 1907 err_free_new_name: 1908 free(new_name); 1909 return -errno; 1910 } 1911 1912 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 1913 { 1914 if (!map || !max_entries) 1915 return -EINVAL; 1916 1917 /* If map already created, its attributes can't be changed. */ 1918 if (map->fd >= 0) 1919 return -EBUSY; 1920 1921 map->def.max_entries = max_entries; 1922 1923 return 0; 1924 } 1925 1926 static int 1927 bpf_object__probe_name(struct bpf_object *obj) 1928 { 1929 struct bpf_load_program_attr attr; 1930 char *cp, errmsg[STRERR_BUFSIZE]; 1931 struct bpf_insn insns[] = { 1932 BPF_MOV64_IMM(BPF_REG_0, 0), 1933 BPF_EXIT_INSN(), 1934 }; 1935 int ret; 1936 1937 /* make sure basic loading works */ 1938 1939 memset(&attr, 0, sizeof(attr)); 1940 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1941 attr.insns = insns; 1942 attr.insns_cnt = ARRAY_SIZE(insns); 1943 attr.license = "GPL"; 1944 1945 ret = bpf_load_program_xattr(&attr, NULL, 0); 1946 if (ret < 0) { 1947 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1948 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", 1949 __func__, cp, errno); 1950 return -errno; 1951 } 1952 close(ret); 1953 1954 /* now try the same program, but with the name */ 1955 1956 attr.name = "test"; 1957 ret = bpf_load_program_xattr(&attr, NULL, 0); 1958 if (ret >= 0) { 1959 obj->caps.name = 1; 1960 close(ret); 1961 } 1962 1963 return 0; 1964 } 1965 1966 static int 1967 bpf_object__probe_global_data(struct bpf_object *obj) 1968 { 1969 struct bpf_load_program_attr prg_attr; 1970 struct bpf_create_map_attr map_attr; 1971 char *cp, errmsg[STRERR_BUFSIZE]; 1972 struct bpf_insn insns[] = { 1973 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 1974 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 1975 BPF_MOV64_IMM(BPF_REG_0, 0), 1976 BPF_EXIT_INSN(), 1977 }; 1978 int ret, map; 1979 1980 memset(&map_attr, 0, sizeof(map_attr)); 1981 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 1982 map_attr.key_size = sizeof(int); 1983 map_attr.value_size = 32; 1984 map_attr.max_entries = 1; 1985 1986 map = bpf_create_map_xattr(&map_attr); 1987 if (map < 0) { 1988 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1989 pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", 1990 __func__, cp, errno); 1991 return -errno; 1992 } 1993 1994 insns[0].imm = map; 1995 1996 memset(&prg_attr, 0, sizeof(prg_attr)); 1997 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1998 prg_attr.insns = insns; 1999 prg_attr.insns_cnt = ARRAY_SIZE(insns); 2000 prg_attr.license = "GPL"; 2001 2002 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 2003 if (ret >= 0) { 2004 obj->caps.global_data = 1; 2005 close(ret); 2006 } 2007 2008 close(map); 2009 return 0; 2010 } 2011 2012 static int bpf_object__probe_btf_func(struct bpf_object *obj) 2013 { 2014 const char strs[] = "\0int\0x\0a"; 2015 /* void x(int a) {} */ 2016 __u32 types[] = { 2017 /* int */ 2018 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 2019 /* FUNC_PROTO */ /* [2] */ 2020 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 2021 BTF_PARAM_ENC(7, 1), 2022 /* FUNC x */ /* [3] */ 2023 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 2024 }; 2025 int btf_fd; 2026 2027 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 2028 strs, sizeof(strs)); 2029 if (btf_fd >= 0) { 2030 obj->caps.btf_func = 1; 2031 close(btf_fd); 2032 return 1; 2033 } 2034 2035 return 0; 2036 } 2037 2038 static int bpf_object__probe_btf_datasec(struct bpf_object *obj) 2039 { 2040 const char strs[] = "\0x\0.data"; 2041 /* static int a; */ 2042 __u32 types[] = { 2043 /* int */ 2044 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 2045 /* VAR x */ /* [2] */ 2046 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 2047 BTF_VAR_STATIC, 2048 /* DATASEC val */ /* [3] */ 2049 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 2050 BTF_VAR_SECINFO_ENC(2, 0, 4), 2051 }; 2052 int btf_fd; 2053 2054 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 2055 strs, sizeof(strs)); 2056 if (btf_fd >= 0) { 2057 obj->caps.btf_datasec = 1; 2058 close(btf_fd); 2059 return 1; 2060 } 2061 2062 return 0; 2063 } 2064 2065 static int 2066 bpf_object__probe_caps(struct bpf_object *obj) 2067 { 2068 int (*probe_fn[])(struct bpf_object *obj) = { 2069 bpf_object__probe_name, 2070 bpf_object__probe_global_data, 2071 bpf_object__probe_btf_func, 2072 bpf_object__probe_btf_datasec, 2073 }; 2074 int i, ret; 2075 2076 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { 2077 ret = probe_fn[i](obj); 2078 if (ret < 0) 2079 pr_debug("Probe #%d failed with %d.\n", i, ret); 2080 } 2081 2082 return 0; 2083 } 2084 2085 static int 2086 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 2087 { 2088 char *cp, errmsg[STRERR_BUFSIZE]; 2089 int err, zero = 0; 2090 __u8 *data; 2091 2092 /* Nothing to do here since kernel already zero-initializes .bss map. */ 2093 if (map->libbpf_type == LIBBPF_MAP_BSS) 2094 return 0; 2095 2096 data = map->libbpf_type == LIBBPF_MAP_DATA ? 2097 obj->sections.data : obj->sections.rodata; 2098 2099 err = bpf_map_update_elem(map->fd, &zero, data, 0); 2100 /* Freeze .rodata map as read-only from syscall side. */ 2101 if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { 2102 err = bpf_map_freeze(map->fd); 2103 if (err) { 2104 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2105 pr_warning("Error freezing map(%s) as read-only: %s\n", 2106 map->name, cp); 2107 err = 0; 2108 } 2109 } 2110 return err; 2111 } 2112 2113 static int 2114 bpf_object__create_maps(struct bpf_object *obj) 2115 { 2116 struct bpf_create_map_attr create_attr = {}; 2117 unsigned int i; 2118 int err; 2119 2120 for (i = 0; i < obj->nr_maps; i++) { 2121 struct bpf_map *map = &obj->maps[i]; 2122 struct bpf_map_def *def = &map->def; 2123 char *cp, errmsg[STRERR_BUFSIZE]; 2124 int *pfd = &map->fd; 2125 2126 if (map->fd >= 0) { 2127 pr_debug("skip map create (preset) %s: fd=%d\n", 2128 map->name, map->fd); 2129 continue; 2130 } 2131 2132 if (obj->caps.name) 2133 create_attr.name = map->name; 2134 create_attr.map_ifindex = map->map_ifindex; 2135 create_attr.map_type = def->type; 2136 create_attr.map_flags = def->map_flags; 2137 create_attr.key_size = def->key_size; 2138 create_attr.value_size = def->value_size; 2139 create_attr.max_entries = def->max_entries; 2140 create_attr.btf_fd = 0; 2141 create_attr.btf_key_type_id = 0; 2142 create_attr.btf_value_type_id = 0; 2143 if (bpf_map_type__is_map_in_map(def->type) && 2144 map->inner_map_fd >= 0) 2145 create_attr.inner_map_fd = map->inner_map_fd; 2146 2147 if (obj->btf && !bpf_map_find_btf_info(obj, map)) { 2148 create_attr.btf_fd = btf__fd(obj->btf); 2149 create_attr.btf_key_type_id = map->btf_key_type_id; 2150 create_attr.btf_value_type_id = map->btf_value_type_id; 2151 } 2152 2153 *pfd = bpf_create_map_xattr(&create_attr); 2154 if (*pfd < 0 && (create_attr.btf_key_type_id || 2155 create_attr.btf_value_type_id)) { 2156 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2157 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 2158 map->name, cp, errno); 2159 create_attr.btf_fd = 0; 2160 create_attr.btf_key_type_id = 0; 2161 create_attr.btf_value_type_id = 0; 2162 map->btf_key_type_id = 0; 2163 map->btf_value_type_id = 0; 2164 *pfd = bpf_create_map_xattr(&create_attr); 2165 } 2166 2167 if (*pfd < 0) { 2168 size_t j; 2169 2170 err = *pfd; 2171 err_out: 2172 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2173 pr_warning("failed to create map (name: '%s'): %s\n", 2174 map->name, cp); 2175 for (j = 0; j < i; j++) 2176 zclose(obj->maps[j].fd); 2177 return err; 2178 } 2179 2180 if (bpf_map__is_internal(map)) { 2181 err = bpf_object__populate_internal_map(obj, map); 2182 if (err < 0) { 2183 zclose(*pfd); 2184 goto err_out; 2185 } 2186 } 2187 2188 pr_debug("created map %s: fd=%d\n", map->name, *pfd); 2189 } 2190 2191 return 0; 2192 } 2193 2194 static int 2195 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 2196 void *btf_prog_info, const char *info_name) 2197 { 2198 if (err != -ENOENT) { 2199 pr_warning("Error in loading %s for sec %s.\n", 2200 info_name, prog->section_name); 2201 return err; 2202 } 2203 2204 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 2205 2206 if (btf_prog_info) { 2207 /* 2208 * Some info has already been found but has problem 2209 * in the last btf_ext reloc. Must have to error out. 2210 */ 2211 pr_warning("Error in relocating %s for sec %s.\n", 2212 info_name, prog->section_name); 2213 return err; 2214 } 2215 2216 /* Have problem loading the very first info. Ignore the rest. */ 2217 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", 2218 info_name, prog->section_name, info_name); 2219 return 0; 2220 } 2221 2222 static int 2223 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 2224 const char *section_name, __u32 insn_offset) 2225 { 2226 int err; 2227 2228 if (!insn_offset || prog->func_info) { 2229 /* 2230 * !insn_offset => main program 2231 * 2232 * For sub prog, the main program's func_info has to 2233 * be loaded first (i.e. prog->func_info != NULL) 2234 */ 2235 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 2236 section_name, insn_offset, 2237 &prog->func_info, 2238 &prog->func_info_cnt); 2239 if (err) 2240 return check_btf_ext_reloc_err(prog, err, 2241 prog->func_info, 2242 "bpf_func_info"); 2243 2244 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 2245 } 2246 2247 if (!insn_offset || prog->line_info) { 2248 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 2249 section_name, insn_offset, 2250 &prog->line_info, 2251 &prog->line_info_cnt); 2252 if (err) 2253 return check_btf_ext_reloc_err(prog, err, 2254 prog->line_info, 2255 "bpf_line_info"); 2256 2257 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 2258 } 2259 2260 if (!insn_offset) 2261 prog->btf_fd = btf__fd(obj->btf); 2262 2263 return 0; 2264 } 2265 2266 static int 2267 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 2268 struct reloc_desc *relo) 2269 { 2270 struct bpf_insn *insn, *new_insn; 2271 struct bpf_program *text; 2272 size_t new_cnt; 2273 int err; 2274 2275 if (relo->type != RELO_CALL) 2276 return -LIBBPF_ERRNO__RELOC; 2277 2278 if (prog->idx == obj->efile.text_shndx) { 2279 pr_warning("relo in .text insn %d into off %d\n", 2280 relo->insn_idx, relo->text_off); 2281 return -LIBBPF_ERRNO__RELOC; 2282 } 2283 2284 if (prog->main_prog_cnt == 0) { 2285 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 2286 if (!text) { 2287 pr_warning("no .text section found yet relo into text exist\n"); 2288 return -LIBBPF_ERRNO__RELOC; 2289 } 2290 new_cnt = prog->insns_cnt + text->insns_cnt; 2291 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 2292 if (!new_insn) { 2293 pr_warning("oom in prog realloc\n"); 2294 return -ENOMEM; 2295 } 2296 2297 if (obj->btf_ext) { 2298 err = bpf_program_reloc_btf_ext(prog, obj, 2299 text->section_name, 2300 prog->insns_cnt); 2301 if (err) 2302 return err; 2303 } 2304 2305 memcpy(new_insn + prog->insns_cnt, text->insns, 2306 text->insns_cnt * sizeof(*insn)); 2307 prog->insns = new_insn; 2308 prog->main_prog_cnt = prog->insns_cnt; 2309 prog->insns_cnt = new_cnt; 2310 pr_debug("added %zd insn from %s to prog %s\n", 2311 text->insns_cnt, text->section_name, 2312 prog->section_name); 2313 } 2314 insn = &prog->insns[relo->insn_idx]; 2315 insn->imm += prog->main_prog_cnt - relo->insn_idx; 2316 return 0; 2317 } 2318 2319 static int 2320 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 2321 { 2322 int i, err; 2323 2324 if (!prog) 2325 return 0; 2326 2327 if (obj->btf_ext) { 2328 err = bpf_program_reloc_btf_ext(prog, obj, 2329 prog->section_name, 0); 2330 if (err) 2331 return err; 2332 } 2333 2334 if (!prog->reloc_desc) 2335 return 0; 2336 2337 for (i = 0; i < prog->nr_reloc; i++) { 2338 if (prog->reloc_desc[i].type == RELO_LD64 || 2339 prog->reloc_desc[i].type == RELO_DATA) { 2340 bool relo_data = prog->reloc_desc[i].type == RELO_DATA; 2341 struct bpf_insn *insns = prog->insns; 2342 int insn_idx, map_idx; 2343 2344 insn_idx = prog->reloc_desc[i].insn_idx; 2345 map_idx = prog->reloc_desc[i].map_idx; 2346 2347 if (insn_idx + 1 >= (int)prog->insns_cnt) { 2348 pr_warning("relocation out of range: '%s'\n", 2349 prog->section_name); 2350 return -LIBBPF_ERRNO__RELOC; 2351 } 2352 2353 if (!relo_data) { 2354 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 2355 } else { 2356 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; 2357 insns[insn_idx + 1].imm = insns[insn_idx].imm; 2358 } 2359 insns[insn_idx].imm = obj->maps[map_idx].fd; 2360 } else if (prog->reloc_desc[i].type == RELO_CALL) { 2361 err = bpf_program__reloc_text(prog, obj, 2362 &prog->reloc_desc[i]); 2363 if (err) 2364 return err; 2365 } 2366 } 2367 2368 zfree(&prog->reloc_desc); 2369 prog->nr_reloc = 0; 2370 return 0; 2371 } 2372 2373 2374 static int 2375 bpf_object__relocate(struct bpf_object *obj) 2376 { 2377 struct bpf_program *prog; 2378 size_t i; 2379 int err; 2380 2381 for (i = 0; i < obj->nr_programs; i++) { 2382 prog = &obj->programs[i]; 2383 2384 err = bpf_program__relocate(prog, obj); 2385 if (err) { 2386 pr_warning("failed to relocate '%s'\n", 2387 prog->section_name); 2388 return err; 2389 } 2390 } 2391 return 0; 2392 } 2393 2394 static int bpf_object__collect_reloc(struct bpf_object *obj) 2395 { 2396 int i, err; 2397 2398 if (!obj_elf_valid(obj)) { 2399 pr_warning("Internal error: elf object is closed\n"); 2400 return -LIBBPF_ERRNO__INTERNAL; 2401 } 2402 2403 for (i = 0; i < obj->efile.nr_reloc; i++) { 2404 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 2405 Elf_Data *data = obj->efile.reloc[i].data; 2406 int idx = shdr->sh_info; 2407 struct bpf_program *prog; 2408 2409 if (shdr->sh_type != SHT_REL) { 2410 pr_warning("internal error at %d\n", __LINE__); 2411 return -LIBBPF_ERRNO__INTERNAL; 2412 } 2413 2414 prog = bpf_object__find_prog_by_idx(obj, idx); 2415 if (!prog) { 2416 pr_warning("relocation failed: no section(%d)\n", idx); 2417 return -LIBBPF_ERRNO__RELOC; 2418 } 2419 2420 err = bpf_program__collect_reloc(prog, shdr, data, obj); 2421 if (err) 2422 return err; 2423 } 2424 return 0; 2425 } 2426 2427 static int 2428 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 2429 char *license, __u32 kern_version, int *pfd) 2430 { 2431 struct bpf_load_program_attr load_attr; 2432 char *cp, errmsg[STRERR_BUFSIZE]; 2433 int log_buf_size = BPF_LOG_BUF_SIZE; 2434 char *log_buf; 2435 int ret; 2436 2437 if (!insns || !insns_cnt) 2438 return -EINVAL; 2439 2440 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 2441 load_attr.prog_type = prog->type; 2442 load_attr.expected_attach_type = prog->expected_attach_type; 2443 if (prog->caps->name) 2444 load_attr.name = prog->name; 2445 load_attr.insns = insns; 2446 load_attr.insns_cnt = insns_cnt; 2447 load_attr.license = license; 2448 load_attr.kern_version = kern_version; 2449 load_attr.prog_ifindex = prog->prog_ifindex; 2450 load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; 2451 load_attr.func_info = prog->func_info; 2452 load_attr.func_info_rec_size = prog->func_info_rec_size; 2453 load_attr.func_info_cnt = prog->func_info_cnt; 2454 load_attr.line_info = prog->line_info; 2455 load_attr.line_info_rec_size = prog->line_info_rec_size; 2456 load_attr.line_info_cnt = prog->line_info_cnt; 2457 load_attr.log_level = prog->log_level; 2458 load_attr.prog_flags = prog->prog_flags; 2459 2460 retry_load: 2461 log_buf = malloc(log_buf_size); 2462 if (!log_buf) 2463 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 2464 2465 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); 2466 2467 if (ret >= 0) { 2468 if (load_attr.log_level) 2469 pr_debug("verifier log:\n%s", log_buf); 2470 *pfd = ret; 2471 ret = 0; 2472 goto out; 2473 } 2474 2475 if (errno == ENOSPC) { 2476 log_buf_size <<= 1; 2477 free(log_buf); 2478 goto retry_load; 2479 } 2480 ret = -LIBBPF_ERRNO__LOAD; 2481 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2482 pr_warning("load bpf program failed: %s\n", cp); 2483 2484 if (log_buf && log_buf[0] != '\0') { 2485 ret = -LIBBPF_ERRNO__VERIFY; 2486 pr_warning("-- BEGIN DUMP LOG ---\n"); 2487 pr_warning("\n%s\n", log_buf); 2488 pr_warning("-- END LOG --\n"); 2489 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 2490 pr_warning("Program too large (%zu insns), at most %d insns\n", 2491 load_attr.insns_cnt, BPF_MAXINSNS); 2492 ret = -LIBBPF_ERRNO__PROG2BIG; 2493 } else { 2494 /* Wrong program type? */ 2495 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 2496 int fd; 2497 2498 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 2499 load_attr.expected_attach_type = 0; 2500 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 2501 if (fd >= 0) { 2502 close(fd); 2503 ret = -LIBBPF_ERRNO__PROGTYPE; 2504 goto out; 2505 } 2506 } 2507 2508 if (log_buf) 2509 ret = -LIBBPF_ERRNO__KVER; 2510 } 2511 2512 out: 2513 free(log_buf); 2514 return ret; 2515 } 2516 2517 int 2518 bpf_program__load(struct bpf_program *prog, 2519 char *license, __u32 kern_version) 2520 { 2521 int err = 0, fd, i; 2522 2523 if (prog->instances.nr < 0 || !prog->instances.fds) { 2524 if (prog->preprocessor) { 2525 pr_warning("Internal error: can't load program '%s'\n", 2526 prog->section_name); 2527 return -LIBBPF_ERRNO__INTERNAL; 2528 } 2529 2530 prog->instances.fds = malloc(sizeof(int)); 2531 if (!prog->instances.fds) { 2532 pr_warning("Not enough memory for BPF fds\n"); 2533 return -ENOMEM; 2534 } 2535 prog->instances.nr = 1; 2536 prog->instances.fds[0] = -1; 2537 } 2538 2539 if (!prog->preprocessor) { 2540 if (prog->instances.nr != 1) { 2541 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 2542 prog->section_name, prog->instances.nr); 2543 } 2544 err = load_program(prog, prog->insns, prog->insns_cnt, 2545 license, kern_version, &fd); 2546 if (!err) 2547 prog->instances.fds[0] = fd; 2548 goto out; 2549 } 2550 2551 for (i = 0; i < prog->instances.nr; i++) { 2552 struct bpf_prog_prep_result result; 2553 bpf_program_prep_t preprocessor = prog->preprocessor; 2554 2555 memset(&result, 0, sizeof(result)); 2556 err = preprocessor(prog, i, prog->insns, 2557 prog->insns_cnt, &result); 2558 if (err) { 2559 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 2560 i, prog->section_name); 2561 goto out; 2562 } 2563 2564 if (!result.new_insn_ptr || !result.new_insn_cnt) { 2565 pr_debug("Skip loading the %dth instance of program '%s'\n", 2566 i, prog->section_name); 2567 prog->instances.fds[i] = -1; 2568 if (result.pfd) 2569 *result.pfd = -1; 2570 continue; 2571 } 2572 2573 err = load_program(prog, result.new_insn_ptr, 2574 result.new_insn_cnt, 2575 license, kern_version, &fd); 2576 2577 if (err) { 2578 pr_warning("Loading the %dth instance of program '%s' failed\n", 2579 i, prog->section_name); 2580 goto out; 2581 } 2582 2583 if (result.pfd) 2584 *result.pfd = fd; 2585 prog->instances.fds[i] = fd; 2586 } 2587 out: 2588 if (err) 2589 pr_warning("failed to load program '%s'\n", 2590 prog->section_name); 2591 zfree(&prog->insns); 2592 prog->insns_cnt = 0; 2593 return err; 2594 } 2595 2596 static bool bpf_program__is_function_storage(const struct bpf_program *prog, 2597 const struct bpf_object *obj) 2598 { 2599 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 2600 } 2601 2602 static int 2603 bpf_object__load_progs(struct bpf_object *obj, int log_level) 2604 { 2605 size_t i; 2606 int err; 2607 2608 for (i = 0; i < obj->nr_programs; i++) { 2609 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 2610 continue; 2611 obj->programs[i].log_level |= log_level; 2612 err = bpf_program__load(&obj->programs[i], 2613 obj->license, 2614 obj->kern_version); 2615 if (err) 2616 return err; 2617 } 2618 return 0; 2619 } 2620 2621 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) 2622 { 2623 switch (type) { 2624 case BPF_PROG_TYPE_SOCKET_FILTER: 2625 case BPF_PROG_TYPE_SCHED_CLS: 2626 case BPF_PROG_TYPE_SCHED_ACT: 2627 case BPF_PROG_TYPE_XDP: 2628 case BPF_PROG_TYPE_CGROUP_SKB: 2629 case BPF_PROG_TYPE_CGROUP_SOCK: 2630 case BPF_PROG_TYPE_LWT_IN: 2631 case BPF_PROG_TYPE_LWT_OUT: 2632 case BPF_PROG_TYPE_LWT_XMIT: 2633 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 2634 case BPF_PROG_TYPE_SOCK_OPS: 2635 case BPF_PROG_TYPE_SK_SKB: 2636 case BPF_PROG_TYPE_CGROUP_DEVICE: 2637 case BPF_PROG_TYPE_SK_MSG: 2638 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2639 case BPF_PROG_TYPE_LIRC_MODE2: 2640 case BPF_PROG_TYPE_SK_REUSEPORT: 2641 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2642 case BPF_PROG_TYPE_UNSPEC: 2643 case BPF_PROG_TYPE_TRACEPOINT: 2644 case BPF_PROG_TYPE_RAW_TRACEPOINT: 2645 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2646 case BPF_PROG_TYPE_PERF_EVENT: 2647 case BPF_PROG_TYPE_CGROUP_SYSCTL: 2648 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2649 return false; 2650 case BPF_PROG_TYPE_KPROBE: 2651 default: 2652 return true; 2653 } 2654 } 2655 2656 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) 2657 { 2658 if (needs_kver && obj->kern_version == 0) { 2659 pr_warning("%s doesn't provide kernel version\n", 2660 obj->path); 2661 return -LIBBPF_ERRNO__KVERSION; 2662 } 2663 return 0; 2664 } 2665 2666 static struct bpf_object * 2667 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, 2668 bool needs_kver, int flags) 2669 { 2670 struct bpf_object *obj; 2671 int err; 2672 2673 if (elf_version(EV_CURRENT) == EV_NONE) { 2674 pr_warning("failed to init libelf for %s\n", path); 2675 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 2676 } 2677 2678 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 2679 if (IS_ERR(obj)) 2680 return obj; 2681 2682 CHECK_ERR(bpf_object__elf_init(obj), err, out); 2683 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 2684 CHECK_ERR(bpf_object__probe_caps(obj), err, out); 2685 CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); 2686 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 2687 CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); 2688 2689 bpf_object__elf_finish(obj); 2690 return obj; 2691 out: 2692 bpf_object__close(obj); 2693 return ERR_PTR(err); 2694 } 2695 2696 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, 2697 int flags) 2698 { 2699 /* param validation */ 2700 if (!attr->file) 2701 return NULL; 2702 2703 pr_debug("loading %s\n", attr->file); 2704 2705 return __bpf_object__open(attr->file, NULL, 0, 2706 bpf_prog_type__needs_kver(attr->prog_type), 2707 flags); 2708 } 2709 2710 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 2711 { 2712 return __bpf_object__open_xattr(attr, 0); 2713 } 2714 2715 struct bpf_object *bpf_object__open(const char *path) 2716 { 2717 struct bpf_object_open_attr attr = { 2718 .file = path, 2719 .prog_type = BPF_PROG_TYPE_UNSPEC, 2720 }; 2721 2722 return bpf_object__open_xattr(&attr); 2723 } 2724 2725 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 2726 size_t obj_buf_sz, 2727 const char *name) 2728 { 2729 char tmp_name[64]; 2730 2731 /* param validation */ 2732 if (!obj_buf || obj_buf_sz <= 0) 2733 return NULL; 2734 2735 if (!name) { 2736 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 2737 (unsigned long)obj_buf, 2738 (unsigned long)obj_buf_sz); 2739 name = tmp_name; 2740 } 2741 pr_debug("loading object '%s' from buffer\n", name); 2742 2743 return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); 2744 } 2745 2746 int bpf_object__unload(struct bpf_object *obj) 2747 { 2748 size_t i; 2749 2750 if (!obj) 2751 return -EINVAL; 2752 2753 for (i = 0; i < obj->nr_maps; i++) 2754 zclose(obj->maps[i].fd); 2755 2756 for (i = 0; i < obj->nr_programs; i++) 2757 bpf_program__unload(&obj->programs[i]); 2758 2759 return 0; 2760 } 2761 2762 int bpf_object__load_xattr(struct bpf_object_load_attr *attr) 2763 { 2764 struct bpf_object *obj; 2765 int err; 2766 2767 if (!attr) 2768 return -EINVAL; 2769 obj = attr->obj; 2770 if (!obj) 2771 return -EINVAL; 2772 2773 if (obj->loaded) { 2774 pr_warning("object should not be loaded twice\n"); 2775 return -EINVAL; 2776 } 2777 2778 obj->loaded = true; 2779 2780 CHECK_ERR(bpf_object__create_maps(obj), err, out); 2781 CHECK_ERR(bpf_object__relocate(obj), err, out); 2782 CHECK_ERR(bpf_object__load_progs(obj, attr->log_level), err, out); 2783 2784 return 0; 2785 out: 2786 bpf_object__unload(obj); 2787 pr_warning("failed to load object '%s'\n", obj->path); 2788 return err; 2789 } 2790 2791 int bpf_object__load(struct bpf_object *obj) 2792 { 2793 struct bpf_object_load_attr attr = { 2794 .obj = obj, 2795 }; 2796 2797 return bpf_object__load_xattr(&attr); 2798 } 2799 2800 static int check_path(const char *path) 2801 { 2802 char *cp, errmsg[STRERR_BUFSIZE]; 2803 struct statfs st_fs; 2804 char *dname, *dir; 2805 int err = 0; 2806 2807 if (path == NULL) 2808 return -EINVAL; 2809 2810 dname = strdup(path); 2811 if (dname == NULL) 2812 return -ENOMEM; 2813 2814 dir = dirname(dname); 2815 if (statfs(dir, &st_fs)) { 2816 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2817 pr_warning("failed to statfs %s: %s\n", dir, cp); 2818 err = -errno; 2819 } 2820 free(dname); 2821 2822 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 2823 pr_warning("specified path %s is not on BPF FS\n", path); 2824 err = -EINVAL; 2825 } 2826 2827 return err; 2828 } 2829 2830 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 2831 int instance) 2832 { 2833 char *cp, errmsg[STRERR_BUFSIZE]; 2834 int err; 2835 2836 err = check_path(path); 2837 if (err) 2838 return err; 2839 2840 if (prog == NULL) { 2841 pr_warning("invalid program pointer\n"); 2842 return -EINVAL; 2843 } 2844 2845 if (instance < 0 || instance >= prog->instances.nr) { 2846 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2847 instance, prog->section_name, prog->instances.nr); 2848 return -EINVAL; 2849 } 2850 2851 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 2852 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2853 pr_warning("failed to pin program: %s\n", cp); 2854 return -errno; 2855 } 2856 pr_debug("pinned program '%s'\n", path); 2857 2858 return 0; 2859 } 2860 2861 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 2862 int instance) 2863 { 2864 int err; 2865 2866 err = check_path(path); 2867 if (err) 2868 return err; 2869 2870 if (prog == NULL) { 2871 pr_warning("invalid program pointer\n"); 2872 return -EINVAL; 2873 } 2874 2875 if (instance < 0 || instance >= prog->instances.nr) { 2876 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2877 instance, prog->section_name, prog->instances.nr); 2878 return -EINVAL; 2879 } 2880 2881 err = unlink(path); 2882 if (err != 0) 2883 return -errno; 2884 pr_debug("unpinned program '%s'\n", path); 2885 2886 return 0; 2887 } 2888 2889 static int make_dir(const char *path) 2890 { 2891 char *cp, errmsg[STRERR_BUFSIZE]; 2892 int err = 0; 2893 2894 if (mkdir(path, 0700) && errno != EEXIST) 2895 err = -errno; 2896 2897 if (err) { 2898 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 2899 pr_warning("failed to mkdir %s: %s\n", path, cp); 2900 } 2901 return err; 2902 } 2903 2904 int bpf_program__pin(struct bpf_program *prog, const char *path) 2905 { 2906 int i, err; 2907 2908 err = check_path(path); 2909 if (err) 2910 return err; 2911 2912 if (prog == NULL) { 2913 pr_warning("invalid program pointer\n"); 2914 return -EINVAL; 2915 } 2916 2917 if (prog->instances.nr <= 0) { 2918 pr_warning("no instances of prog %s to pin\n", 2919 prog->section_name); 2920 return -EINVAL; 2921 } 2922 2923 if (prog->instances.nr == 1) { 2924 /* don't create subdirs when pinning single instance */ 2925 return bpf_program__pin_instance(prog, path, 0); 2926 } 2927 2928 err = make_dir(path); 2929 if (err) 2930 return err; 2931 2932 for (i = 0; i < prog->instances.nr; i++) { 2933 char buf[PATH_MAX]; 2934 int len; 2935 2936 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2937 if (len < 0) { 2938 err = -EINVAL; 2939 goto err_unpin; 2940 } else if (len >= PATH_MAX) { 2941 err = -ENAMETOOLONG; 2942 goto err_unpin; 2943 } 2944 2945 err = bpf_program__pin_instance(prog, buf, i); 2946 if (err) 2947 goto err_unpin; 2948 } 2949 2950 return 0; 2951 2952 err_unpin: 2953 for (i = i - 1; i >= 0; i--) { 2954 char buf[PATH_MAX]; 2955 int len; 2956 2957 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2958 if (len < 0) 2959 continue; 2960 else if (len >= PATH_MAX) 2961 continue; 2962 2963 bpf_program__unpin_instance(prog, buf, i); 2964 } 2965 2966 rmdir(path); 2967 2968 return err; 2969 } 2970 2971 int bpf_program__unpin(struct bpf_program *prog, const char *path) 2972 { 2973 int i, err; 2974 2975 err = check_path(path); 2976 if (err) 2977 return err; 2978 2979 if (prog == NULL) { 2980 pr_warning("invalid program pointer\n"); 2981 return -EINVAL; 2982 } 2983 2984 if (prog->instances.nr <= 0) { 2985 pr_warning("no instances of prog %s to pin\n", 2986 prog->section_name); 2987 return -EINVAL; 2988 } 2989 2990 if (prog->instances.nr == 1) { 2991 /* don't create subdirs when pinning single instance */ 2992 return bpf_program__unpin_instance(prog, path, 0); 2993 } 2994 2995 for (i = 0; i < prog->instances.nr; i++) { 2996 char buf[PATH_MAX]; 2997 int len; 2998 2999 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 3000 if (len < 0) 3001 return -EINVAL; 3002 else if (len >= PATH_MAX) 3003 return -ENAMETOOLONG; 3004 3005 err = bpf_program__unpin_instance(prog, buf, i); 3006 if (err) 3007 return err; 3008 } 3009 3010 err = rmdir(path); 3011 if (err) 3012 return -errno; 3013 3014 return 0; 3015 } 3016 3017 int bpf_map__pin(struct bpf_map *map, const char *path) 3018 { 3019 char *cp, errmsg[STRERR_BUFSIZE]; 3020 int err; 3021 3022 err = check_path(path); 3023 if (err) 3024 return err; 3025 3026 if (map == NULL) { 3027 pr_warning("invalid map pointer\n"); 3028 return -EINVAL; 3029 } 3030 3031 if (bpf_obj_pin(map->fd, path)) { 3032 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 3033 pr_warning("failed to pin map: %s\n", cp); 3034 return -errno; 3035 } 3036 3037 pr_debug("pinned map '%s'\n", path); 3038 3039 return 0; 3040 } 3041 3042 int bpf_map__unpin(struct bpf_map *map, const char *path) 3043 { 3044 int err; 3045 3046 err = check_path(path); 3047 if (err) 3048 return err; 3049 3050 if (map == NULL) { 3051 pr_warning("invalid map pointer\n"); 3052 return -EINVAL; 3053 } 3054 3055 err = unlink(path); 3056 if (err != 0) 3057 return -errno; 3058 pr_debug("unpinned map '%s'\n", path); 3059 3060 return 0; 3061 } 3062 3063 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 3064 { 3065 struct bpf_map *map; 3066 int err; 3067 3068 if (!obj) 3069 return -ENOENT; 3070 3071 if (!obj->loaded) { 3072 pr_warning("object not yet loaded; load it first\n"); 3073 return -ENOENT; 3074 } 3075 3076 err = make_dir(path); 3077 if (err) 3078 return err; 3079 3080 bpf_object__for_each_map(map, obj) { 3081 char buf[PATH_MAX]; 3082 int len; 3083 3084 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3085 bpf_map__name(map)); 3086 if (len < 0) { 3087 err = -EINVAL; 3088 goto err_unpin_maps; 3089 } else if (len >= PATH_MAX) { 3090 err = -ENAMETOOLONG; 3091 goto err_unpin_maps; 3092 } 3093 3094 err = bpf_map__pin(map, buf); 3095 if (err) 3096 goto err_unpin_maps; 3097 } 3098 3099 return 0; 3100 3101 err_unpin_maps: 3102 while ((map = bpf_map__prev(map, obj))) { 3103 char buf[PATH_MAX]; 3104 int len; 3105 3106 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3107 bpf_map__name(map)); 3108 if (len < 0) 3109 continue; 3110 else if (len >= PATH_MAX) 3111 continue; 3112 3113 bpf_map__unpin(map, buf); 3114 } 3115 3116 return err; 3117 } 3118 3119 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 3120 { 3121 struct bpf_map *map; 3122 int err; 3123 3124 if (!obj) 3125 return -ENOENT; 3126 3127 bpf_object__for_each_map(map, obj) { 3128 char buf[PATH_MAX]; 3129 int len; 3130 3131 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3132 bpf_map__name(map)); 3133 if (len < 0) 3134 return -EINVAL; 3135 else if (len >= PATH_MAX) 3136 return -ENAMETOOLONG; 3137 3138 err = bpf_map__unpin(map, buf); 3139 if (err) 3140 return err; 3141 } 3142 3143 return 0; 3144 } 3145 3146 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 3147 { 3148 struct bpf_program *prog; 3149 int err; 3150 3151 if (!obj) 3152 return -ENOENT; 3153 3154 if (!obj->loaded) { 3155 pr_warning("object not yet loaded; load it first\n"); 3156 return -ENOENT; 3157 } 3158 3159 err = make_dir(path); 3160 if (err) 3161 return err; 3162 3163 bpf_object__for_each_program(prog, obj) { 3164 char buf[PATH_MAX]; 3165 int len; 3166 3167 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3168 prog->pin_name); 3169 if (len < 0) { 3170 err = -EINVAL; 3171 goto err_unpin_programs; 3172 } else if (len >= PATH_MAX) { 3173 err = -ENAMETOOLONG; 3174 goto err_unpin_programs; 3175 } 3176 3177 err = bpf_program__pin(prog, buf); 3178 if (err) 3179 goto err_unpin_programs; 3180 } 3181 3182 return 0; 3183 3184 err_unpin_programs: 3185 while ((prog = bpf_program__prev(prog, obj))) { 3186 char buf[PATH_MAX]; 3187 int len; 3188 3189 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3190 prog->pin_name); 3191 if (len < 0) 3192 continue; 3193 else if (len >= PATH_MAX) 3194 continue; 3195 3196 bpf_program__unpin(prog, buf); 3197 } 3198 3199 return err; 3200 } 3201 3202 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 3203 { 3204 struct bpf_program *prog; 3205 int err; 3206 3207 if (!obj) 3208 return -ENOENT; 3209 3210 bpf_object__for_each_program(prog, obj) { 3211 char buf[PATH_MAX]; 3212 int len; 3213 3214 len = snprintf(buf, PATH_MAX, "%s/%s", path, 3215 prog->pin_name); 3216 if (len < 0) 3217 return -EINVAL; 3218 else if (len >= PATH_MAX) 3219 return -ENAMETOOLONG; 3220 3221 err = bpf_program__unpin(prog, buf); 3222 if (err) 3223 return err; 3224 } 3225 3226 return 0; 3227 } 3228 3229 int bpf_object__pin(struct bpf_object *obj, const char *path) 3230 { 3231 int err; 3232 3233 err = bpf_object__pin_maps(obj, path); 3234 if (err) 3235 return err; 3236 3237 err = bpf_object__pin_programs(obj, path); 3238 if (err) { 3239 bpf_object__unpin_maps(obj, path); 3240 return err; 3241 } 3242 3243 return 0; 3244 } 3245 3246 void bpf_object__close(struct bpf_object *obj) 3247 { 3248 size_t i; 3249 3250 if (!obj) 3251 return; 3252 3253 if (obj->clear_priv) 3254 obj->clear_priv(obj, obj->priv); 3255 3256 bpf_object__elf_finish(obj); 3257 bpf_object__unload(obj); 3258 btf__free(obj->btf); 3259 btf_ext__free(obj->btf_ext); 3260 3261 for (i = 0; i < obj->nr_maps; i++) { 3262 zfree(&obj->maps[i].name); 3263 if (obj->maps[i].clear_priv) 3264 obj->maps[i].clear_priv(&obj->maps[i], 3265 obj->maps[i].priv); 3266 obj->maps[i].priv = NULL; 3267 obj->maps[i].clear_priv = NULL; 3268 } 3269 3270 zfree(&obj->sections.rodata); 3271 zfree(&obj->sections.data); 3272 zfree(&obj->maps); 3273 obj->nr_maps = 0; 3274 3275 if (obj->programs && obj->nr_programs) { 3276 for (i = 0; i < obj->nr_programs; i++) 3277 bpf_program__exit(&obj->programs[i]); 3278 } 3279 zfree(&obj->programs); 3280 3281 list_del(&obj->list); 3282 free(obj); 3283 } 3284 3285 struct bpf_object * 3286 bpf_object__next(struct bpf_object *prev) 3287 { 3288 struct bpf_object *next; 3289 3290 if (!prev) 3291 next = list_first_entry(&bpf_objects_list, 3292 struct bpf_object, 3293 list); 3294 else 3295 next = list_next_entry(prev, list); 3296 3297 /* Empty list is noticed here so don't need checking on entry. */ 3298 if (&next->list == &bpf_objects_list) 3299 return NULL; 3300 3301 return next; 3302 } 3303 3304 const char *bpf_object__name(const struct bpf_object *obj) 3305 { 3306 return obj ? obj->path : ERR_PTR(-EINVAL); 3307 } 3308 3309 unsigned int bpf_object__kversion(const struct bpf_object *obj) 3310 { 3311 return obj ? obj->kern_version : 0; 3312 } 3313 3314 struct btf *bpf_object__btf(const struct bpf_object *obj) 3315 { 3316 return obj ? obj->btf : NULL; 3317 } 3318 3319 int bpf_object__btf_fd(const struct bpf_object *obj) 3320 { 3321 return obj->btf ? btf__fd(obj->btf) : -1; 3322 } 3323 3324 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 3325 bpf_object_clear_priv_t clear_priv) 3326 { 3327 if (obj->priv && obj->clear_priv) 3328 obj->clear_priv(obj, obj->priv); 3329 3330 obj->priv = priv; 3331 obj->clear_priv = clear_priv; 3332 return 0; 3333 } 3334 3335 void *bpf_object__priv(const struct bpf_object *obj) 3336 { 3337 return obj ? obj->priv : ERR_PTR(-EINVAL); 3338 } 3339 3340 static struct bpf_program * 3341 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 3342 bool forward) 3343 { 3344 size_t nr_programs = obj->nr_programs; 3345 ssize_t idx; 3346 3347 if (!nr_programs) 3348 return NULL; 3349 3350 if (!p) 3351 /* Iter from the beginning */ 3352 return forward ? &obj->programs[0] : 3353 &obj->programs[nr_programs - 1]; 3354 3355 if (p->obj != obj) { 3356 pr_warning("error: program handler doesn't match object\n"); 3357 return NULL; 3358 } 3359 3360 idx = (p - obj->programs) + (forward ? 1 : -1); 3361 if (idx >= obj->nr_programs || idx < 0) 3362 return NULL; 3363 return &obj->programs[idx]; 3364 } 3365 3366 struct bpf_program * 3367 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) 3368 { 3369 struct bpf_program *prog = prev; 3370 3371 do { 3372 prog = __bpf_program__iter(prog, obj, true); 3373 } while (prog && bpf_program__is_function_storage(prog, obj)); 3374 3375 return prog; 3376 } 3377 3378 struct bpf_program * 3379 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) 3380 { 3381 struct bpf_program *prog = next; 3382 3383 do { 3384 prog = __bpf_program__iter(prog, obj, false); 3385 } while (prog && bpf_program__is_function_storage(prog, obj)); 3386 3387 return prog; 3388 } 3389 3390 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 3391 bpf_program_clear_priv_t clear_priv) 3392 { 3393 if (prog->priv && prog->clear_priv) 3394 prog->clear_priv(prog, prog->priv); 3395 3396 prog->priv = priv; 3397 prog->clear_priv = clear_priv; 3398 return 0; 3399 } 3400 3401 void *bpf_program__priv(const struct bpf_program *prog) 3402 { 3403 return prog ? prog->priv : ERR_PTR(-EINVAL); 3404 } 3405 3406 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 3407 { 3408 prog->prog_ifindex = ifindex; 3409 } 3410 3411 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) 3412 { 3413 const char *title; 3414 3415 title = prog->section_name; 3416 if (needs_copy) { 3417 title = strdup(title); 3418 if (!title) { 3419 pr_warning("failed to strdup program title\n"); 3420 return ERR_PTR(-ENOMEM); 3421 } 3422 } 3423 3424 return title; 3425 } 3426 3427 int bpf_program__fd(const struct bpf_program *prog) 3428 { 3429 return bpf_program__nth_fd(prog, 0); 3430 } 3431 3432 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 3433 bpf_program_prep_t prep) 3434 { 3435 int *instances_fds; 3436 3437 if (nr_instances <= 0 || !prep) 3438 return -EINVAL; 3439 3440 if (prog->instances.nr > 0 || prog->instances.fds) { 3441 pr_warning("Can't set pre-processor after loading\n"); 3442 return -EINVAL; 3443 } 3444 3445 instances_fds = malloc(sizeof(int) * nr_instances); 3446 if (!instances_fds) { 3447 pr_warning("alloc memory failed for fds\n"); 3448 return -ENOMEM; 3449 } 3450 3451 /* fill all fd with -1 */ 3452 memset(instances_fds, -1, sizeof(int) * nr_instances); 3453 3454 prog->instances.nr = nr_instances; 3455 prog->instances.fds = instances_fds; 3456 prog->preprocessor = prep; 3457 return 0; 3458 } 3459 3460 int bpf_program__nth_fd(const struct bpf_program *prog, int n) 3461 { 3462 int fd; 3463 3464 if (!prog) 3465 return -EINVAL; 3466 3467 if (n >= prog->instances.nr || n < 0) { 3468 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 3469 n, prog->section_name, prog->instances.nr); 3470 return -EINVAL; 3471 } 3472 3473 fd = prog->instances.fds[n]; 3474 if (fd < 0) { 3475 pr_warning("%dth instance of program '%s' is invalid\n", 3476 n, prog->section_name); 3477 return -ENOENT; 3478 } 3479 3480 return fd; 3481 } 3482 3483 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 3484 { 3485 prog->type = type; 3486 } 3487 3488 static bool bpf_program__is_type(const struct bpf_program *prog, 3489 enum bpf_prog_type type) 3490 { 3491 return prog ? (prog->type == type) : false; 3492 } 3493 3494 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 3495 int bpf_program__set_##NAME(struct bpf_program *prog) \ 3496 { \ 3497 if (!prog) \ 3498 return -EINVAL; \ 3499 bpf_program__set_type(prog, TYPE); \ 3500 return 0; \ 3501 } \ 3502 \ 3503 bool bpf_program__is_##NAME(const struct bpf_program *prog) \ 3504 { \ 3505 return bpf_program__is_type(prog, TYPE); \ 3506 } \ 3507 3508 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 3509 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 3510 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 3511 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 3512 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 3513 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 3514 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 3515 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 3516 3517 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 3518 enum bpf_attach_type type) 3519 { 3520 prog->expected_attach_type = type; 3521 } 3522 3523 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ 3524 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } 3525 3526 /* Programs that can NOT be attached. */ 3527 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) 3528 3529 /* Programs that can be attached. */ 3530 #define BPF_APROG_SEC(string, ptype, atype) \ 3531 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) 3532 3533 /* Programs that must specify expected attach type at load time. */ 3534 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 3535 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) 3536 3537 /* Programs that can be attached but attach type can't be identified by section 3538 * name. Kept for backward compatibility. 3539 */ 3540 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 3541 3542 static const struct { 3543 const char *sec; 3544 size_t len; 3545 enum bpf_prog_type prog_type; 3546 enum bpf_attach_type expected_attach_type; 3547 int is_attachable; 3548 enum bpf_attach_type attach_type; 3549 } section_names[] = { 3550 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 3551 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 3552 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 3553 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 3554 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 3555 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 3556 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), 3557 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 3558 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 3559 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 3560 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 3561 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 3562 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 3563 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 3564 BPF_CGROUP_INET_INGRESS), 3565 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 3566 BPF_CGROUP_INET_EGRESS), 3567 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 3568 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 3569 BPF_CGROUP_INET_SOCK_CREATE), 3570 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 3571 BPF_CGROUP_INET4_POST_BIND), 3572 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 3573 BPF_CGROUP_INET6_POST_BIND), 3574 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 3575 BPF_CGROUP_DEVICE), 3576 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 3577 BPF_CGROUP_SOCK_OPS), 3578 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 3579 BPF_SK_SKB_STREAM_PARSER), 3580 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 3581 BPF_SK_SKB_STREAM_VERDICT), 3582 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 3583 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 3584 BPF_SK_MSG_VERDICT), 3585 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 3586 BPF_LIRC_MODE2), 3587 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 3588 BPF_FLOW_DISSECTOR), 3589 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3590 BPF_CGROUP_INET4_BIND), 3591 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3592 BPF_CGROUP_INET6_BIND), 3593 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3594 BPF_CGROUP_INET4_CONNECT), 3595 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3596 BPF_CGROUP_INET6_CONNECT), 3597 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3598 BPF_CGROUP_UDP4_SENDMSG), 3599 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3600 BPF_CGROUP_UDP6_SENDMSG), 3601 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3602 BPF_CGROUP_UDP4_RECVMSG), 3603 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3604 BPF_CGROUP_UDP6_RECVMSG), 3605 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 3606 BPF_CGROUP_SYSCTL), 3607 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 3608 BPF_CGROUP_GETSOCKOPT), 3609 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, 3610 BPF_CGROUP_SETSOCKOPT), 3611 }; 3612 3613 #undef BPF_PROG_SEC_IMPL 3614 #undef BPF_PROG_SEC 3615 #undef BPF_APROG_SEC 3616 #undef BPF_EAPROG_SEC 3617 #undef BPF_APROG_COMPAT 3618 3619 #define MAX_TYPE_NAME_SIZE 32 3620 3621 static char *libbpf_get_type_names(bool attach_type) 3622 { 3623 int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; 3624 char *buf; 3625 3626 buf = malloc(len); 3627 if (!buf) 3628 return NULL; 3629 3630 buf[0] = '\0'; 3631 /* Forge string buf with all available names */ 3632 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3633 if (attach_type && !section_names[i].is_attachable) 3634 continue; 3635 3636 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { 3637 free(buf); 3638 return NULL; 3639 } 3640 strcat(buf, " "); 3641 strcat(buf, section_names[i].sec); 3642 } 3643 3644 return buf; 3645 } 3646 3647 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 3648 enum bpf_attach_type *expected_attach_type) 3649 { 3650 char *type_names; 3651 int i; 3652 3653 if (!name) 3654 return -EINVAL; 3655 3656 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3657 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3658 continue; 3659 *prog_type = section_names[i].prog_type; 3660 *expected_attach_type = section_names[i].expected_attach_type; 3661 return 0; 3662 } 3663 pr_warning("failed to guess program type based on ELF section name '%s'\n", name); 3664 type_names = libbpf_get_type_names(false); 3665 if (type_names != NULL) { 3666 pr_info("supported section(type) names are:%s\n", type_names); 3667 free(type_names); 3668 } 3669 3670 return -EINVAL; 3671 } 3672 3673 int libbpf_attach_type_by_name(const char *name, 3674 enum bpf_attach_type *attach_type) 3675 { 3676 char *type_names; 3677 int i; 3678 3679 if (!name) 3680 return -EINVAL; 3681 3682 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3683 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3684 continue; 3685 if (!section_names[i].is_attachable) 3686 return -EINVAL; 3687 *attach_type = section_names[i].attach_type; 3688 return 0; 3689 } 3690 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); 3691 type_names = libbpf_get_type_names(true); 3692 if (type_names != NULL) { 3693 pr_info("attachable section(type) names are:%s\n", type_names); 3694 free(type_names); 3695 } 3696 3697 return -EINVAL; 3698 } 3699 3700 static int 3701 bpf_program__identify_section(struct bpf_program *prog, 3702 enum bpf_prog_type *prog_type, 3703 enum bpf_attach_type *expected_attach_type) 3704 { 3705 return libbpf_prog_type_by_name(prog->section_name, prog_type, 3706 expected_attach_type); 3707 } 3708 3709 int bpf_map__fd(const struct bpf_map *map) 3710 { 3711 return map ? map->fd : -EINVAL; 3712 } 3713 3714 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) 3715 { 3716 return map ? &map->def : ERR_PTR(-EINVAL); 3717 } 3718 3719 const char *bpf_map__name(const struct bpf_map *map) 3720 { 3721 return map ? map->name : NULL; 3722 } 3723 3724 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 3725 { 3726 return map ? map->btf_key_type_id : 0; 3727 } 3728 3729 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 3730 { 3731 return map ? map->btf_value_type_id : 0; 3732 } 3733 3734 int bpf_map__set_priv(struct bpf_map *map, void *priv, 3735 bpf_map_clear_priv_t clear_priv) 3736 { 3737 if (!map) 3738 return -EINVAL; 3739 3740 if (map->priv) { 3741 if (map->clear_priv) 3742 map->clear_priv(map, map->priv); 3743 } 3744 3745 map->priv = priv; 3746 map->clear_priv = clear_priv; 3747 return 0; 3748 } 3749 3750 void *bpf_map__priv(const struct bpf_map *map) 3751 { 3752 return map ? map->priv : ERR_PTR(-EINVAL); 3753 } 3754 3755 bool bpf_map__is_offload_neutral(const struct bpf_map *map) 3756 { 3757 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 3758 } 3759 3760 bool bpf_map__is_internal(const struct bpf_map *map) 3761 { 3762 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 3763 } 3764 3765 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 3766 { 3767 map->map_ifindex = ifindex; 3768 } 3769 3770 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 3771 { 3772 if (!bpf_map_type__is_map_in_map(map->def.type)) { 3773 pr_warning("error: unsupported map type\n"); 3774 return -EINVAL; 3775 } 3776 if (map->inner_map_fd != -1) { 3777 pr_warning("error: inner_map_fd already specified\n"); 3778 return -EINVAL; 3779 } 3780 map->inner_map_fd = fd; 3781 return 0; 3782 } 3783 3784 static struct bpf_map * 3785 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 3786 { 3787 ssize_t idx; 3788 struct bpf_map *s, *e; 3789 3790 if (!obj || !obj->maps) 3791 return NULL; 3792 3793 s = obj->maps; 3794 e = obj->maps + obj->nr_maps; 3795 3796 if ((m < s) || (m >= e)) { 3797 pr_warning("error in %s: map handler doesn't belong to object\n", 3798 __func__); 3799 return NULL; 3800 } 3801 3802 idx = (m - obj->maps) + i; 3803 if (idx >= obj->nr_maps || idx < 0) 3804 return NULL; 3805 return &obj->maps[idx]; 3806 } 3807 3808 struct bpf_map * 3809 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) 3810 { 3811 if (prev == NULL) 3812 return obj->maps; 3813 3814 return __bpf_map__iter(prev, obj, 1); 3815 } 3816 3817 struct bpf_map * 3818 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) 3819 { 3820 if (next == NULL) { 3821 if (!obj->nr_maps) 3822 return NULL; 3823 return obj->maps + obj->nr_maps - 1; 3824 } 3825 3826 return __bpf_map__iter(next, obj, -1); 3827 } 3828 3829 struct bpf_map * 3830 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 3831 { 3832 struct bpf_map *pos; 3833 3834 bpf_object__for_each_map(pos, obj) { 3835 if (pos->name && !strcmp(pos->name, name)) 3836 return pos; 3837 } 3838 return NULL; 3839 } 3840 3841 int 3842 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 3843 { 3844 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 3845 } 3846 3847 struct bpf_map * 3848 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 3849 { 3850 return ERR_PTR(-ENOTSUP); 3851 } 3852 3853 long libbpf_get_error(const void *ptr) 3854 { 3855 return PTR_ERR_OR_ZERO(ptr); 3856 } 3857 3858 int bpf_prog_load(const char *file, enum bpf_prog_type type, 3859 struct bpf_object **pobj, int *prog_fd) 3860 { 3861 struct bpf_prog_load_attr attr; 3862 3863 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 3864 attr.file = file; 3865 attr.prog_type = type; 3866 attr.expected_attach_type = 0; 3867 3868 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 3869 } 3870 3871 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 3872 struct bpf_object **pobj, int *prog_fd) 3873 { 3874 struct bpf_object_open_attr open_attr = {}; 3875 struct bpf_program *prog, *first_prog = NULL; 3876 enum bpf_attach_type expected_attach_type; 3877 enum bpf_prog_type prog_type; 3878 struct bpf_object *obj; 3879 struct bpf_map *map; 3880 int err; 3881 3882 if (!attr) 3883 return -EINVAL; 3884 if (!attr->file) 3885 return -EINVAL; 3886 3887 open_attr.file = attr->file; 3888 open_attr.prog_type = attr->prog_type; 3889 3890 obj = bpf_object__open_xattr(&open_attr); 3891 if (IS_ERR_OR_NULL(obj)) 3892 return -ENOENT; 3893 3894 bpf_object__for_each_program(prog, obj) { 3895 /* 3896 * If type is not specified, try to guess it based on 3897 * section name. 3898 */ 3899 prog_type = attr->prog_type; 3900 prog->prog_ifindex = attr->ifindex; 3901 expected_attach_type = attr->expected_attach_type; 3902 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 3903 err = bpf_program__identify_section(prog, &prog_type, 3904 &expected_attach_type); 3905 if (err < 0) { 3906 bpf_object__close(obj); 3907 return -EINVAL; 3908 } 3909 } 3910 3911 bpf_program__set_type(prog, prog_type); 3912 bpf_program__set_expected_attach_type(prog, 3913 expected_attach_type); 3914 3915 prog->log_level = attr->log_level; 3916 prog->prog_flags = attr->prog_flags; 3917 if (!first_prog) 3918 first_prog = prog; 3919 } 3920 3921 bpf_object__for_each_map(map, obj) { 3922 if (!bpf_map__is_offload_neutral(map)) 3923 map->map_ifindex = attr->ifindex; 3924 } 3925 3926 if (!first_prog) { 3927 pr_warning("object file doesn't contain bpf program\n"); 3928 bpf_object__close(obj); 3929 return -ENOENT; 3930 } 3931 3932 err = bpf_object__load(obj); 3933 if (err) { 3934 bpf_object__close(obj); 3935 return -EINVAL; 3936 } 3937 3938 *pobj = obj; 3939 *prog_fd = bpf_program__fd(first_prog); 3940 return 0; 3941 } 3942 3943 struct bpf_link { 3944 int (*destroy)(struct bpf_link *link); 3945 }; 3946 3947 int bpf_link__destroy(struct bpf_link *link) 3948 { 3949 int err; 3950 3951 if (!link) 3952 return 0; 3953 3954 err = link->destroy(link); 3955 free(link); 3956 3957 return err; 3958 } 3959 3960 struct bpf_link_fd { 3961 struct bpf_link link; /* has to be at the top of struct */ 3962 int fd; /* hook FD */ 3963 }; 3964 3965 static int bpf_link__destroy_perf_event(struct bpf_link *link) 3966 { 3967 struct bpf_link_fd *l = (void *)link; 3968 int err; 3969 3970 err = ioctl(l->fd, PERF_EVENT_IOC_DISABLE, 0); 3971 if (err) 3972 err = -errno; 3973 3974 close(l->fd); 3975 return err; 3976 } 3977 3978 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, 3979 int pfd) 3980 { 3981 char errmsg[STRERR_BUFSIZE]; 3982 struct bpf_link_fd *link; 3983 int prog_fd, err; 3984 3985 if (pfd < 0) { 3986 pr_warning("program '%s': invalid perf event FD %d\n", 3987 bpf_program__title(prog, false), pfd); 3988 return ERR_PTR(-EINVAL); 3989 } 3990 prog_fd = bpf_program__fd(prog); 3991 if (prog_fd < 0) { 3992 pr_warning("program '%s': can't attach BPF program w/o FD (did you load it?)\n", 3993 bpf_program__title(prog, false)); 3994 return ERR_PTR(-EINVAL); 3995 } 3996 3997 link = malloc(sizeof(*link)); 3998 if (!link) 3999 return ERR_PTR(-ENOMEM); 4000 link->link.destroy = &bpf_link__destroy_perf_event; 4001 link->fd = pfd; 4002 4003 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 4004 err = -errno; 4005 free(link); 4006 pr_warning("program '%s': failed to attach to pfd %d: %s\n", 4007 bpf_program__title(prog, false), pfd, 4008 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4009 return ERR_PTR(err); 4010 } 4011 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 4012 err = -errno; 4013 free(link); 4014 pr_warning("program '%s': failed to enable pfd %d: %s\n", 4015 bpf_program__title(prog, false), pfd, 4016 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4017 return ERR_PTR(err); 4018 } 4019 return (struct bpf_link *)link; 4020 } 4021 4022 /* 4023 * this function is expected to parse integer in the range of [0, 2^31-1] from 4024 * given file using scanf format string fmt. If actual parsed value is 4025 * negative, the result might be indistinguishable from error 4026 */ 4027 static int parse_uint_from_file(const char *file, const char *fmt) 4028 { 4029 char buf[STRERR_BUFSIZE]; 4030 int err, ret; 4031 FILE *f; 4032 4033 f = fopen(file, "r"); 4034 if (!f) { 4035 err = -errno; 4036 pr_debug("failed to open '%s': %s\n", file, 4037 libbpf_strerror_r(err, buf, sizeof(buf))); 4038 return err; 4039 } 4040 err = fscanf(f, fmt, &ret); 4041 if (err != 1) { 4042 err = err == EOF ? -EIO : -errno; 4043 pr_debug("failed to parse '%s': %s\n", file, 4044 libbpf_strerror_r(err, buf, sizeof(buf))); 4045 fclose(f); 4046 return err; 4047 } 4048 fclose(f); 4049 return ret; 4050 } 4051 4052 static int determine_kprobe_perf_type(void) 4053 { 4054 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 4055 4056 return parse_uint_from_file(file, "%d\n"); 4057 } 4058 4059 static int determine_uprobe_perf_type(void) 4060 { 4061 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 4062 4063 return parse_uint_from_file(file, "%d\n"); 4064 } 4065 4066 static int determine_kprobe_retprobe_bit(void) 4067 { 4068 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 4069 4070 return parse_uint_from_file(file, "config:%d\n"); 4071 } 4072 4073 static int determine_uprobe_retprobe_bit(void) 4074 { 4075 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 4076 4077 return parse_uint_from_file(file, "config:%d\n"); 4078 } 4079 4080 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 4081 uint64_t offset, int pid) 4082 { 4083 struct perf_event_attr attr = {}; 4084 char errmsg[STRERR_BUFSIZE]; 4085 int type, pfd, err; 4086 4087 type = uprobe ? determine_uprobe_perf_type() 4088 : determine_kprobe_perf_type(); 4089 if (type < 0) { 4090 pr_warning("failed to determine %s perf type: %s\n", 4091 uprobe ? "uprobe" : "kprobe", 4092 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 4093 return type; 4094 } 4095 if (retprobe) { 4096 int bit = uprobe ? determine_uprobe_retprobe_bit() 4097 : determine_kprobe_retprobe_bit(); 4098 4099 if (bit < 0) { 4100 pr_warning("failed to determine %s retprobe bit: %s\n", 4101 uprobe ? "uprobe" : "kprobe", 4102 libbpf_strerror_r(bit, errmsg, 4103 sizeof(errmsg))); 4104 return bit; 4105 } 4106 attr.config |= 1 << bit; 4107 } 4108 attr.size = sizeof(attr); 4109 attr.type = type; 4110 attr.config1 = (uint64_t)(void *)name; /* kprobe_func or uprobe_path */ 4111 attr.config2 = offset; /* kprobe_addr or probe_offset */ 4112 4113 /* pid filter is meaningful only for uprobes */ 4114 pfd = syscall(__NR_perf_event_open, &attr, 4115 pid < 0 ? -1 : pid /* pid */, 4116 pid == -1 ? 0 : -1 /* cpu */, 4117 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 4118 if (pfd < 0) { 4119 err = -errno; 4120 pr_warning("%s perf_event_open() failed: %s\n", 4121 uprobe ? "uprobe" : "kprobe", 4122 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4123 return err; 4124 } 4125 return pfd; 4126 } 4127 4128 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, 4129 bool retprobe, 4130 const char *func_name) 4131 { 4132 char errmsg[STRERR_BUFSIZE]; 4133 struct bpf_link *link; 4134 int pfd, err; 4135 4136 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, 4137 0 /* offset */, -1 /* pid */); 4138 if (pfd < 0) { 4139 pr_warning("program '%s': failed to create %s '%s' perf event: %s\n", 4140 bpf_program__title(prog, false), 4141 retprobe ? "kretprobe" : "kprobe", func_name, 4142 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4143 return ERR_PTR(pfd); 4144 } 4145 link = bpf_program__attach_perf_event(prog, pfd); 4146 if (IS_ERR(link)) { 4147 close(pfd); 4148 err = PTR_ERR(link); 4149 pr_warning("program '%s': failed to attach to %s '%s': %s\n", 4150 bpf_program__title(prog, false), 4151 retprobe ? "kretprobe" : "kprobe", func_name, 4152 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4153 return link; 4154 } 4155 return link; 4156 } 4157 4158 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, 4159 bool retprobe, pid_t pid, 4160 const char *binary_path, 4161 size_t func_offset) 4162 { 4163 char errmsg[STRERR_BUFSIZE]; 4164 struct bpf_link *link; 4165 int pfd, err; 4166 4167 pfd = perf_event_open_probe(true /* uprobe */, retprobe, 4168 binary_path, func_offset, pid); 4169 if (pfd < 0) { 4170 pr_warning("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 4171 bpf_program__title(prog, false), 4172 retprobe ? "uretprobe" : "uprobe", 4173 binary_path, func_offset, 4174 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4175 return ERR_PTR(pfd); 4176 } 4177 link = bpf_program__attach_perf_event(prog, pfd); 4178 if (IS_ERR(link)) { 4179 close(pfd); 4180 err = PTR_ERR(link); 4181 pr_warning("program '%s': failed to attach to %s '%s:0x%zx': %s\n", 4182 bpf_program__title(prog, false), 4183 retprobe ? "uretprobe" : "uprobe", 4184 binary_path, func_offset, 4185 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4186 return link; 4187 } 4188 return link; 4189 } 4190 4191 static int determine_tracepoint_id(const char *tp_category, 4192 const char *tp_name) 4193 { 4194 char file[PATH_MAX]; 4195 int ret; 4196 4197 ret = snprintf(file, sizeof(file), 4198 "/sys/kernel/debug/tracing/events/%s/%s/id", 4199 tp_category, tp_name); 4200 if (ret < 0) 4201 return -errno; 4202 if (ret >= sizeof(file)) { 4203 pr_debug("tracepoint %s/%s path is too long\n", 4204 tp_category, tp_name); 4205 return -E2BIG; 4206 } 4207 return parse_uint_from_file(file, "%d\n"); 4208 } 4209 4210 static int perf_event_open_tracepoint(const char *tp_category, 4211 const char *tp_name) 4212 { 4213 struct perf_event_attr attr = {}; 4214 char errmsg[STRERR_BUFSIZE]; 4215 int tp_id, pfd, err; 4216 4217 tp_id = determine_tracepoint_id(tp_category, tp_name); 4218 if (tp_id < 0) { 4219 pr_warning("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 4220 tp_category, tp_name, 4221 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 4222 return tp_id; 4223 } 4224 4225 attr.type = PERF_TYPE_TRACEPOINT; 4226 attr.size = sizeof(attr); 4227 attr.config = tp_id; 4228 4229 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 4230 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 4231 if (pfd < 0) { 4232 err = -errno; 4233 pr_warning("tracepoint '%s/%s' perf_event_open() failed: %s\n", 4234 tp_category, tp_name, 4235 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4236 return err; 4237 } 4238 return pfd; 4239 } 4240 4241 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, 4242 const char *tp_category, 4243 const char *tp_name) 4244 { 4245 char errmsg[STRERR_BUFSIZE]; 4246 struct bpf_link *link; 4247 int pfd, err; 4248 4249 pfd = perf_event_open_tracepoint(tp_category, tp_name); 4250 if (pfd < 0) { 4251 pr_warning("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 4252 bpf_program__title(prog, false), 4253 tp_category, tp_name, 4254 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4255 return ERR_PTR(pfd); 4256 } 4257 link = bpf_program__attach_perf_event(prog, pfd); 4258 if (IS_ERR(link)) { 4259 close(pfd); 4260 err = PTR_ERR(link); 4261 pr_warning("program '%s': failed to attach to tracepoint '%s/%s': %s\n", 4262 bpf_program__title(prog, false), 4263 tp_category, tp_name, 4264 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 4265 return link; 4266 } 4267 return link; 4268 } 4269 4270 static int bpf_link__destroy_fd(struct bpf_link *link) 4271 { 4272 struct bpf_link_fd *l = (void *)link; 4273 4274 return close(l->fd); 4275 } 4276 4277 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, 4278 const char *tp_name) 4279 { 4280 char errmsg[STRERR_BUFSIZE]; 4281 struct bpf_link_fd *link; 4282 int prog_fd, pfd; 4283 4284 prog_fd = bpf_program__fd(prog); 4285 if (prog_fd < 0) { 4286 pr_warning("program '%s': can't attach before loaded\n", 4287 bpf_program__title(prog, false)); 4288 return ERR_PTR(-EINVAL); 4289 } 4290 4291 link = malloc(sizeof(*link)); 4292 if (!link) 4293 return ERR_PTR(-ENOMEM); 4294 link->link.destroy = &bpf_link__destroy_fd; 4295 4296 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 4297 if (pfd < 0) { 4298 pfd = -errno; 4299 free(link); 4300 pr_warning("program '%s': failed to attach to raw tracepoint '%s': %s\n", 4301 bpf_program__title(prog, false), tp_name, 4302 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 4303 return ERR_PTR(pfd); 4304 } 4305 link->fd = pfd; 4306 return (struct bpf_link *)link; 4307 } 4308 4309 enum bpf_perf_event_ret 4310 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 4311 void **copy_mem, size_t *copy_size, 4312 bpf_perf_event_print_t fn, void *private_data) 4313 { 4314 struct perf_event_mmap_page *header = mmap_mem; 4315 __u64 data_head = ring_buffer_read_head(header); 4316 __u64 data_tail = header->data_tail; 4317 void *base = ((__u8 *)header) + page_size; 4318 int ret = LIBBPF_PERF_EVENT_CONT; 4319 struct perf_event_header *ehdr; 4320 size_t ehdr_size; 4321 4322 while (data_head != data_tail) { 4323 ehdr = base + (data_tail & (mmap_size - 1)); 4324 ehdr_size = ehdr->size; 4325 4326 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 4327 void *copy_start = ehdr; 4328 size_t len_first = base + mmap_size - copy_start; 4329 size_t len_secnd = ehdr_size - len_first; 4330 4331 if (*copy_size < ehdr_size) { 4332 free(*copy_mem); 4333 *copy_mem = malloc(ehdr_size); 4334 if (!*copy_mem) { 4335 *copy_size = 0; 4336 ret = LIBBPF_PERF_EVENT_ERROR; 4337 break; 4338 } 4339 *copy_size = ehdr_size; 4340 } 4341 4342 memcpy(*copy_mem, copy_start, len_first); 4343 memcpy(*copy_mem + len_first, base, len_secnd); 4344 ehdr = *copy_mem; 4345 } 4346 4347 ret = fn(ehdr, private_data); 4348 data_tail += ehdr_size; 4349 if (ret != LIBBPF_PERF_EVENT_CONT) 4350 break; 4351 } 4352 4353 ring_buffer_write_tail(header, data_tail); 4354 return ret; 4355 } 4356 4357 struct bpf_prog_info_array_desc { 4358 int array_offset; /* e.g. offset of jited_prog_insns */ 4359 int count_offset; /* e.g. offset of jited_prog_len */ 4360 int size_offset; /* > 0: offset of rec size, 4361 * < 0: fix size of -size_offset 4362 */ 4363 }; 4364 4365 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 4366 [BPF_PROG_INFO_JITED_INSNS] = { 4367 offsetof(struct bpf_prog_info, jited_prog_insns), 4368 offsetof(struct bpf_prog_info, jited_prog_len), 4369 -1, 4370 }, 4371 [BPF_PROG_INFO_XLATED_INSNS] = { 4372 offsetof(struct bpf_prog_info, xlated_prog_insns), 4373 offsetof(struct bpf_prog_info, xlated_prog_len), 4374 -1, 4375 }, 4376 [BPF_PROG_INFO_MAP_IDS] = { 4377 offsetof(struct bpf_prog_info, map_ids), 4378 offsetof(struct bpf_prog_info, nr_map_ids), 4379 -(int)sizeof(__u32), 4380 }, 4381 [BPF_PROG_INFO_JITED_KSYMS] = { 4382 offsetof(struct bpf_prog_info, jited_ksyms), 4383 offsetof(struct bpf_prog_info, nr_jited_ksyms), 4384 -(int)sizeof(__u64), 4385 }, 4386 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 4387 offsetof(struct bpf_prog_info, jited_func_lens), 4388 offsetof(struct bpf_prog_info, nr_jited_func_lens), 4389 -(int)sizeof(__u32), 4390 }, 4391 [BPF_PROG_INFO_FUNC_INFO] = { 4392 offsetof(struct bpf_prog_info, func_info), 4393 offsetof(struct bpf_prog_info, nr_func_info), 4394 offsetof(struct bpf_prog_info, func_info_rec_size), 4395 }, 4396 [BPF_PROG_INFO_LINE_INFO] = { 4397 offsetof(struct bpf_prog_info, line_info), 4398 offsetof(struct bpf_prog_info, nr_line_info), 4399 offsetof(struct bpf_prog_info, line_info_rec_size), 4400 }, 4401 [BPF_PROG_INFO_JITED_LINE_INFO] = { 4402 offsetof(struct bpf_prog_info, jited_line_info), 4403 offsetof(struct bpf_prog_info, nr_jited_line_info), 4404 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 4405 }, 4406 [BPF_PROG_INFO_PROG_TAGS] = { 4407 offsetof(struct bpf_prog_info, prog_tags), 4408 offsetof(struct bpf_prog_info, nr_prog_tags), 4409 -(int)sizeof(__u8) * BPF_TAG_SIZE, 4410 }, 4411 4412 }; 4413 4414 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) 4415 { 4416 __u32 *array = (__u32 *)info; 4417 4418 if (offset >= 0) 4419 return array[offset / sizeof(__u32)]; 4420 return -(int)offset; 4421 } 4422 4423 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) 4424 { 4425 __u64 *array = (__u64 *)info; 4426 4427 if (offset >= 0) 4428 return array[offset / sizeof(__u64)]; 4429 return -(int)offset; 4430 } 4431 4432 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 4433 __u32 val) 4434 { 4435 __u32 *array = (__u32 *)info; 4436 4437 if (offset >= 0) 4438 array[offset / sizeof(__u32)] = val; 4439 } 4440 4441 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 4442 __u64 val) 4443 { 4444 __u64 *array = (__u64 *)info; 4445 4446 if (offset >= 0) 4447 array[offset / sizeof(__u64)] = val; 4448 } 4449 4450 struct bpf_prog_info_linear * 4451 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 4452 { 4453 struct bpf_prog_info_linear *info_linear; 4454 struct bpf_prog_info info = {}; 4455 __u32 info_len = sizeof(info); 4456 __u32 data_len = 0; 4457 int i, err; 4458 void *ptr; 4459 4460 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 4461 return ERR_PTR(-EINVAL); 4462 4463 /* step 1: get array dimensions */ 4464 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 4465 if (err) { 4466 pr_debug("can't get prog info: %s", strerror(errno)); 4467 return ERR_PTR(-EFAULT); 4468 } 4469 4470 /* step 2: calculate total size of all arrays */ 4471 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4472 bool include_array = (arrays & (1UL << i)) > 0; 4473 struct bpf_prog_info_array_desc *desc; 4474 __u32 count, size; 4475 4476 desc = bpf_prog_info_array_desc + i; 4477 4478 /* kernel is too old to support this field */ 4479 if (info_len < desc->array_offset + sizeof(__u32) || 4480 info_len < desc->count_offset + sizeof(__u32) || 4481 (desc->size_offset > 0 && info_len < desc->size_offset)) 4482 include_array = false; 4483 4484 if (!include_array) { 4485 arrays &= ~(1UL << i); /* clear the bit */ 4486 continue; 4487 } 4488 4489 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 4490 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 4491 4492 data_len += count * size; 4493 } 4494 4495 /* step 3: allocate continuous memory */ 4496 data_len = roundup(data_len, sizeof(__u64)); 4497 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 4498 if (!info_linear) 4499 return ERR_PTR(-ENOMEM); 4500 4501 /* step 4: fill data to info_linear->info */ 4502 info_linear->arrays = arrays; 4503 memset(&info_linear->info, 0, sizeof(info)); 4504 ptr = info_linear->data; 4505 4506 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4507 struct bpf_prog_info_array_desc *desc; 4508 __u32 count, size; 4509 4510 if ((arrays & (1UL << i)) == 0) 4511 continue; 4512 4513 desc = bpf_prog_info_array_desc + i; 4514 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 4515 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 4516 bpf_prog_info_set_offset_u32(&info_linear->info, 4517 desc->count_offset, count); 4518 bpf_prog_info_set_offset_u32(&info_linear->info, 4519 desc->size_offset, size); 4520 bpf_prog_info_set_offset_u64(&info_linear->info, 4521 desc->array_offset, 4522 ptr_to_u64(ptr)); 4523 ptr += count * size; 4524 } 4525 4526 /* step 5: call syscall again to get required arrays */ 4527 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 4528 if (err) { 4529 pr_debug("can't get prog info: %s", strerror(errno)); 4530 free(info_linear); 4531 return ERR_PTR(-EFAULT); 4532 } 4533 4534 /* step 6: verify the data */ 4535 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4536 struct bpf_prog_info_array_desc *desc; 4537 __u32 v1, v2; 4538 4539 if ((arrays & (1UL << i)) == 0) 4540 continue; 4541 4542 desc = bpf_prog_info_array_desc + i; 4543 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 4544 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 4545 desc->count_offset); 4546 if (v1 != v2) 4547 pr_warning("%s: mismatch in element count\n", __func__); 4548 4549 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 4550 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 4551 desc->size_offset); 4552 if (v1 != v2) 4553 pr_warning("%s: mismatch in rec size\n", __func__); 4554 } 4555 4556 /* step 7: update info_len and data_len */ 4557 info_linear->info_len = sizeof(struct bpf_prog_info); 4558 info_linear->data_len = data_len; 4559 4560 return info_linear; 4561 } 4562 4563 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 4564 { 4565 int i; 4566 4567 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4568 struct bpf_prog_info_array_desc *desc; 4569 __u64 addr, offs; 4570 4571 if ((info_linear->arrays & (1UL << i)) == 0) 4572 continue; 4573 4574 desc = bpf_prog_info_array_desc + i; 4575 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 4576 desc->array_offset); 4577 offs = addr - ptr_to_u64(info_linear->data); 4578 bpf_prog_info_set_offset_u64(&info_linear->info, 4579 desc->array_offset, offs); 4580 } 4581 } 4582 4583 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 4584 { 4585 int i; 4586 4587 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 4588 struct bpf_prog_info_array_desc *desc; 4589 __u64 addr, offs; 4590 4591 if ((info_linear->arrays & (1UL << i)) == 0) 4592 continue; 4593 4594 desc = bpf_prog_info_array_desc + i; 4595 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 4596 desc->array_offset); 4597 addr = offs + ptr_to_u64(info_linear->data); 4598 bpf_prog_info_set_offset_u64(&info_linear->info, 4599 desc->array_offset, addr); 4600 } 4601 } 4602 4603 int libbpf_num_possible_cpus(void) 4604 { 4605 static const char *fcpu = "/sys/devices/system/cpu/possible"; 4606 int len = 0, n = 0, il = 0, ir = 0; 4607 unsigned int start = 0, end = 0; 4608 static int cpus; 4609 char buf[128]; 4610 int error = 0; 4611 int fd = -1; 4612 4613 if (cpus > 0) 4614 return cpus; 4615 4616 fd = open(fcpu, O_RDONLY); 4617 if (fd < 0) { 4618 error = errno; 4619 pr_warning("Failed to open file %s: %s\n", 4620 fcpu, strerror(error)); 4621 return -error; 4622 } 4623 len = read(fd, buf, sizeof(buf)); 4624 close(fd); 4625 if (len <= 0) { 4626 error = len ? errno : EINVAL; 4627 pr_warning("Failed to read # of possible cpus from %s: %s\n", 4628 fcpu, strerror(error)); 4629 return -error; 4630 } 4631 if (len == sizeof(buf)) { 4632 pr_warning("File %s size overflow\n", fcpu); 4633 return -EOVERFLOW; 4634 } 4635 buf[len] = '\0'; 4636 4637 for (ir = 0, cpus = 0; ir <= len; ir++) { 4638 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 4639 if (buf[ir] == ',' || buf[ir] == '\0') { 4640 buf[ir] = '\0'; 4641 n = sscanf(&buf[il], "%u-%u", &start, &end); 4642 if (n <= 0) { 4643 pr_warning("Failed to get # CPUs from %s\n", 4644 &buf[il]); 4645 return -EINVAL; 4646 } else if (n == 1) { 4647 end = start; 4648 } 4649 cpus += end - start + 1; 4650 il = ir + 1; 4651 } 4652 } 4653 if (cpus <= 0) { 4654 pr_warning("Invalid #CPUs %d from %s\n", cpus, fcpu); 4655 return -EINVAL; 4656 } 4657 return cpus; 4658 } 4659