1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <string.h> 22 #include <unistd.h> 23 #include <fcntl.h> 24 #include <errno.h> 25 #include <asm/unistd.h> 26 #include <linux/err.h> 27 #include <linux/kernel.h> 28 #include <linux/bpf.h> 29 #include <linux/btf.h> 30 #include <linux/filter.h> 31 #include <linux/list.h> 32 #include <linux/limits.h> 33 #include <linux/perf_event.h> 34 #include <linux/ring_buffer.h> 35 #include <sys/stat.h> 36 #include <sys/types.h> 37 #include <sys/vfs.h> 38 #include <tools/libc_compat.h> 39 #include <libelf.h> 40 #include <gelf.h> 41 42 #include "libbpf.h" 43 #include "bpf.h" 44 #include "btf.h" 45 #include "str_error.h" 46 #include "libbpf_internal.h" 47 48 #ifndef EM_BPF 49 #define EM_BPF 247 50 #endif 51 52 #ifndef BPF_FS_MAGIC 53 #define BPF_FS_MAGIC 0xcafe4a11 54 #endif 55 56 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 57 * compilation if user enables corresponding warning. Disable it explicitly. 58 */ 59 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 60 61 #define __printf(a, b) __attribute__((format(printf, a, b))) 62 63 static int __base_pr(enum libbpf_print_level level, const char *format, 64 va_list args) 65 { 66 if (level == LIBBPF_DEBUG) 67 return 0; 68 69 return vfprintf(stderr, format, args); 70 } 71 72 static libbpf_print_fn_t __libbpf_pr = __base_pr; 73 74 void libbpf_set_print(libbpf_print_fn_t fn) 75 { 76 __libbpf_pr = fn; 77 } 78 79 __printf(2, 3) 80 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 81 { 82 va_list args; 83 84 if (!__libbpf_pr) 85 return; 86 87 va_start(args, format); 88 __libbpf_pr(level, format, args); 89 va_end(args); 90 } 91 92 #define STRERR_BUFSIZE 128 93 94 #define CHECK_ERR(action, err, out) do { \ 95 err = action; \ 96 if (err) \ 97 goto out; \ 98 } while(0) 99 100 101 /* Copied from tools/perf/util/util.h */ 102 #ifndef zfree 103 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 104 #endif 105 106 #ifndef zclose 107 # define zclose(fd) ({ \ 108 int ___err = 0; \ 109 if ((fd) >= 0) \ 110 ___err = close((fd)); \ 111 fd = -1; \ 112 ___err; }) 113 #endif 114 115 #ifdef HAVE_LIBELF_MMAP_SUPPORT 116 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 117 #else 118 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 119 #endif 120 121 static inline __u64 ptr_to_u64(const void *ptr) 122 { 123 return (__u64) (unsigned long) ptr; 124 } 125 126 struct bpf_capabilities { 127 /* v4.14: kernel support for program & map names. */ 128 __u32 name:1; 129 /* v5.2: kernel support for global data sections. */ 130 __u32 global_data:1; 131 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 132 __u32 btf_func:1; 133 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 134 __u32 btf_datasec:1; 135 }; 136 137 /* 138 * bpf_prog should be a better name but it has been used in 139 * linux/filter.h. 140 */ 141 struct bpf_program { 142 /* Index in elf obj file, for relocation use. */ 143 int idx; 144 char *name; 145 int prog_ifindex; 146 char *section_name; 147 /* section_name with / replaced by _; makes recursive pinning 148 * in bpf_object__pin_programs easier 149 */ 150 char *pin_name; 151 struct bpf_insn *insns; 152 size_t insns_cnt, main_prog_cnt; 153 enum bpf_prog_type type; 154 155 struct reloc_desc { 156 enum { 157 RELO_LD64, 158 RELO_CALL, 159 RELO_DATA, 160 } type; 161 int insn_idx; 162 union { 163 int map_idx; 164 int text_off; 165 }; 166 } *reloc_desc; 167 int nr_reloc; 168 int log_level; 169 170 struct { 171 int nr; 172 int *fds; 173 } instances; 174 bpf_program_prep_t preprocessor; 175 176 struct bpf_object *obj; 177 void *priv; 178 bpf_program_clear_priv_t clear_priv; 179 180 enum bpf_attach_type expected_attach_type; 181 int btf_fd; 182 void *func_info; 183 __u32 func_info_rec_size; 184 __u32 func_info_cnt; 185 186 struct bpf_capabilities *caps; 187 188 void *line_info; 189 __u32 line_info_rec_size; 190 __u32 line_info_cnt; 191 __u32 prog_flags; 192 }; 193 194 enum libbpf_map_type { 195 LIBBPF_MAP_UNSPEC, 196 LIBBPF_MAP_DATA, 197 LIBBPF_MAP_BSS, 198 LIBBPF_MAP_RODATA, 199 }; 200 201 static const char * const libbpf_type_to_btf_name[] = { 202 [LIBBPF_MAP_DATA] = ".data", 203 [LIBBPF_MAP_BSS] = ".bss", 204 [LIBBPF_MAP_RODATA] = ".rodata", 205 }; 206 207 struct bpf_map { 208 int fd; 209 char *name; 210 size_t offset; 211 int map_ifindex; 212 int inner_map_fd; 213 struct bpf_map_def def; 214 __u32 btf_key_type_id; 215 __u32 btf_value_type_id; 216 void *priv; 217 bpf_map_clear_priv_t clear_priv; 218 enum libbpf_map_type libbpf_type; 219 }; 220 221 struct bpf_secdata { 222 void *rodata; 223 void *data; 224 }; 225 226 static LIST_HEAD(bpf_objects_list); 227 228 struct bpf_object { 229 char name[BPF_OBJ_NAME_LEN]; 230 char license[64]; 231 __u32 kern_version; 232 233 struct bpf_program *programs; 234 size_t nr_programs; 235 struct bpf_map *maps; 236 size_t nr_maps; 237 struct bpf_secdata sections; 238 239 bool loaded; 240 bool has_pseudo_calls; 241 242 /* 243 * Information when doing elf related work. Only valid if fd 244 * is valid. 245 */ 246 struct { 247 int fd; 248 void *obj_buf; 249 size_t obj_buf_sz; 250 Elf *elf; 251 GElf_Ehdr ehdr; 252 Elf_Data *symbols; 253 Elf_Data *data; 254 Elf_Data *rodata; 255 Elf_Data *bss; 256 size_t strtabidx; 257 struct { 258 GElf_Shdr shdr; 259 Elf_Data *data; 260 } *reloc; 261 int nr_reloc; 262 int maps_shndx; 263 int text_shndx; 264 int data_shndx; 265 int rodata_shndx; 266 int bss_shndx; 267 } efile; 268 /* 269 * All loaded bpf_object is linked in a list, which is 270 * hidden to caller. bpf_objects__<func> handlers deal with 271 * all objects. 272 */ 273 struct list_head list; 274 275 struct btf *btf; 276 struct btf_ext *btf_ext; 277 278 void *priv; 279 bpf_object_clear_priv_t clear_priv; 280 281 struct bpf_capabilities caps; 282 283 char path[]; 284 }; 285 #define obj_elf_valid(o) ((o)->efile.elf) 286 287 void bpf_program__unload(struct bpf_program *prog) 288 { 289 int i; 290 291 if (!prog) 292 return; 293 294 /* 295 * If the object is opened but the program was never loaded, 296 * it is possible that prog->instances.nr == -1. 297 */ 298 if (prog->instances.nr > 0) { 299 for (i = 0; i < prog->instances.nr; i++) 300 zclose(prog->instances.fds[i]); 301 } else if (prog->instances.nr != -1) { 302 pr_warning("Internal error: instances.nr is %d\n", 303 prog->instances.nr); 304 } 305 306 prog->instances.nr = -1; 307 zfree(&prog->instances.fds); 308 309 zclose(prog->btf_fd); 310 zfree(&prog->func_info); 311 zfree(&prog->line_info); 312 } 313 314 static void bpf_program__exit(struct bpf_program *prog) 315 { 316 if (!prog) 317 return; 318 319 if (prog->clear_priv) 320 prog->clear_priv(prog, prog->priv); 321 322 prog->priv = NULL; 323 prog->clear_priv = NULL; 324 325 bpf_program__unload(prog); 326 zfree(&prog->name); 327 zfree(&prog->section_name); 328 zfree(&prog->pin_name); 329 zfree(&prog->insns); 330 zfree(&prog->reloc_desc); 331 332 prog->nr_reloc = 0; 333 prog->insns_cnt = 0; 334 prog->idx = -1; 335 } 336 337 static char *__bpf_program__pin_name(struct bpf_program *prog) 338 { 339 char *name, *p; 340 341 name = p = strdup(prog->section_name); 342 while ((p = strchr(p, '/'))) 343 *p = '_'; 344 345 return name; 346 } 347 348 static int 349 bpf_program__init(void *data, size_t size, char *section_name, int idx, 350 struct bpf_program *prog) 351 { 352 const size_t bpf_insn_sz = sizeof(struct bpf_insn); 353 354 if (size == 0 || size % bpf_insn_sz) { 355 pr_warning("corrupted section '%s', size: %zu\n", 356 section_name, size); 357 return -EINVAL; 358 } 359 360 memset(prog, 0, sizeof(*prog)); 361 362 prog->section_name = strdup(section_name); 363 if (!prog->section_name) { 364 pr_warning("failed to alloc name for prog under section(%d) %s\n", 365 idx, section_name); 366 goto errout; 367 } 368 369 prog->pin_name = __bpf_program__pin_name(prog); 370 if (!prog->pin_name) { 371 pr_warning("failed to alloc pin name for prog under section(%d) %s\n", 372 idx, section_name); 373 goto errout; 374 } 375 376 prog->insns = malloc(size); 377 if (!prog->insns) { 378 pr_warning("failed to alloc insns for prog under section %s\n", 379 section_name); 380 goto errout; 381 } 382 prog->insns_cnt = size / bpf_insn_sz; 383 memcpy(prog->insns, data, size); 384 prog->idx = idx; 385 prog->instances.fds = NULL; 386 prog->instances.nr = -1; 387 prog->type = BPF_PROG_TYPE_UNSPEC; 388 prog->btf_fd = -1; 389 390 return 0; 391 errout: 392 bpf_program__exit(prog); 393 return -ENOMEM; 394 } 395 396 static int 397 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 398 char *section_name, int idx) 399 { 400 struct bpf_program prog, *progs; 401 int nr_progs, err; 402 403 err = bpf_program__init(data, size, section_name, idx, &prog); 404 if (err) 405 return err; 406 407 prog.caps = &obj->caps; 408 progs = obj->programs; 409 nr_progs = obj->nr_programs; 410 411 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 412 if (!progs) { 413 /* 414 * In this case the original obj->programs 415 * is still valid, so don't need special treat for 416 * bpf_close_object(). 417 */ 418 pr_warning("failed to alloc a new program under section '%s'\n", 419 section_name); 420 bpf_program__exit(&prog); 421 return -ENOMEM; 422 } 423 424 pr_debug("found program %s\n", prog.section_name); 425 obj->programs = progs; 426 obj->nr_programs = nr_progs + 1; 427 prog.obj = obj; 428 progs[nr_progs] = prog; 429 return 0; 430 } 431 432 static int 433 bpf_object__init_prog_names(struct bpf_object *obj) 434 { 435 Elf_Data *symbols = obj->efile.symbols; 436 struct bpf_program *prog; 437 size_t pi, si; 438 439 for (pi = 0; pi < obj->nr_programs; pi++) { 440 const char *name = NULL; 441 442 prog = &obj->programs[pi]; 443 444 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 445 si++) { 446 GElf_Sym sym; 447 448 if (!gelf_getsym(symbols, si, &sym)) 449 continue; 450 if (sym.st_shndx != prog->idx) 451 continue; 452 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 453 continue; 454 455 name = elf_strptr(obj->efile.elf, 456 obj->efile.strtabidx, 457 sym.st_name); 458 if (!name) { 459 pr_warning("failed to get sym name string for prog %s\n", 460 prog->section_name); 461 return -LIBBPF_ERRNO__LIBELF; 462 } 463 } 464 465 if (!name && prog->idx == obj->efile.text_shndx) 466 name = ".text"; 467 468 if (!name) { 469 pr_warning("failed to find sym for prog %s\n", 470 prog->section_name); 471 return -EINVAL; 472 } 473 474 prog->name = strdup(name); 475 if (!prog->name) { 476 pr_warning("failed to allocate memory for prog sym %s\n", 477 name); 478 return -ENOMEM; 479 } 480 } 481 482 return 0; 483 } 484 485 static struct bpf_object *bpf_object__new(const char *path, 486 void *obj_buf, 487 size_t obj_buf_sz) 488 { 489 struct bpf_object *obj; 490 char *end; 491 492 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 493 if (!obj) { 494 pr_warning("alloc memory failed for %s\n", path); 495 return ERR_PTR(-ENOMEM); 496 } 497 498 strcpy(obj->path, path); 499 /* Using basename() GNU version which doesn't modify arg. */ 500 strncpy(obj->name, basename((void *)path), sizeof(obj->name) - 1); 501 end = strchr(obj->name, '.'); 502 if (end) 503 *end = 0; 504 505 obj->efile.fd = -1; 506 /* 507 * Caller of this function should also call 508 * bpf_object__elf_finish() after data collection to return 509 * obj_buf to user. If not, we should duplicate the buffer to 510 * avoid user freeing them before elf finish. 511 */ 512 obj->efile.obj_buf = obj_buf; 513 obj->efile.obj_buf_sz = obj_buf_sz; 514 obj->efile.maps_shndx = -1; 515 obj->efile.data_shndx = -1; 516 obj->efile.rodata_shndx = -1; 517 obj->efile.bss_shndx = -1; 518 519 obj->loaded = false; 520 521 INIT_LIST_HEAD(&obj->list); 522 list_add(&obj->list, &bpf_objects_list); 523 return obj; 524 } 525 526 static void bpf_object__elf_finish(struct bpf_object *obj) 527 { 528 if (!obj_elf_valid(obj)) 529 return; 530 531 if (obj->efile.elf) { 532 elf_end(obj->efile.elf); 533 obj->efile.elf = NULL; 534 } 535 obj->efile.symbols = NULL; 536 obj->efile.data = NULL; 537 obj->efile.rodata = NULL; 538 obj->efile.bss = NULL; 539 540 zfree(&obj->efile.reloc); 541 obj->efile.nr_reloc = 0; 542 zclose(obj->efile.fd); 543 obj->efile.obj_buf = NULL; 544 obj->efile.obj_buf_sz = 0; 545 } 546 547 static int bpf_object__elf_init(struct bpf_object *obj) 548 { 549 int err = 0; 550 GElf_Ehdr *ep; 551 552 if (obj_elf_valid(obj)) { 553 pr_warning("elf init: internal error\n"); 554 return -LIBBPF_ERRNO__LIBELF; 555 } 556 557 if (obj->efile.obj_buf_sz > 0) { 558 /* 559 * obj_buf should have been validated by 560 * bpf_object__open_buffer(). 561 */ 562 obj->efile.elf = elf_memory(obj->efile.obj_buf, 563 obj->efile.obj_buf_sz); 564 } else { 565 obj->efile.fd = open(obj->path, O_RDONLY); 566 if (obj->efile.fd < 0) { 567 char errmsg[STRERR_BUFSIZE], *cp; 568 569 err = -errno; 570 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 571 pr_warning("failed to open %s: %s\n", obj->path, cp); 572 return err; 573 } 574 575 obj->efile.elf = elf_begin(obj->efile.fd, 576 LIBBPF_ELF_C_READ_MMAP, NULL); 577 } 578 579 if (!obj->efile.elf) { 580 pr_warning("failed to open %s as ELF file\n", obj->path); 581 err = -LIBBPF_ERRNO__LIBELF; 582 goto errout; 583 } 584 585 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 586 pr_warning("failed to get EHDR from %s\n", obj->path); 587 err = -LIBBPF_ERRNO__FORMAT; 588 goto errout; 589 } 590 ep = &obj->efile.ehdr; 591 592 /* Old LLVM set e_machine to EM_NONE */ 593 if (ep->e_type != ET_REL || 594 (ep->e_machine && ep->e_machine != EM_BPF)) { 595 pr_warning("%s is not an eBPF object file\n", obj->path); 596 err = -LIBBPF_ERRNO__FORMAT; 597 goto errout; 598 } 599 600 return 0; 601 errout: 602 bpf_object__elf_finish(obj); 603 return err; 604 } 605 606 static int bpf_object__check_endianness(struct bpf_object *obj) 607 { 608 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 609 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) 610 return 0; 611 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 612 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) 613 return 0; 614 #else 615 # error "Unrecognized __BYTE_ORDER__" 616 #endif 617 pr_warning("endianness mismatch.\n"); 618 return -LIBBPF_ERRNO__ENDIAN; 619 } 620 621 static int 622 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 623 { 624 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); 625 pr_debug("license of %s is %s\n", obj->path, obj->license); 626 return 0; 627 } 628 629 static int 630 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 631 { 632 __u32 kver; 633 634 if (size != sizeof(kver)) { 635 pr_warning("invalid kver section in %s\n", obj->path); 636 return -LIBBPF_ERRNO__FORMAT; 637 } 638 memcpy(&kver, data, sizeof(kver)); 639 obj->kern_version = kver; 640 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 641 return 0; 642 } 643 644 static int compare_bpf_map(const void *_a, const void *_b) 645 { 646 const struct bpf_map *a = _a; 647 const struct bpf_map *b = _b; 648 649 return a->offset - b->offset; 650 } 651 652 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 653 { 654 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 655 type == BPF_MAP_TYPE_HASH_OF_MAPS) 656 return true; 657 return false; 658 } 659 660 static int bpf_object_search_section_size(const struct bpf_object *obj, 661 const char *name, size_t *d_size) 662 { 663 const GElf_Ehdr *ep = &obj->efile.ehdr; 664 Elf *elf = obj->efile.elf; 665 Elf_Scn *scn = NULL; 666 int idx = 0; 667 668 while ((scn = elf_nextscn(elf, scn)) != NULL) { 669 const char *sec_name; 670 Elf_Data *data; 671 GElf_Shdr sh; 672 673 idx++; 674 if (gelf_getshdr(scn, &sh) != &sh) { 675 pr_warning("failed to get section(%d) header from %s\n", 676 idx, obj->path); 677 return -EIO; 678 } 679 680 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 681 if (!sec_name) { 682 pr_warning("failed to get section(%d) name from %s\n", 683 idx, obj->path); 684 return -EIO; 685 } 686 687 if (strcmp(name, sec_name)) 688 continue; 689 690 data = elf_getdata(scn, 0); 691 if (!data) { 692 pr_warning("failed to get section(%d) data from %s(%s)\n", 693 idx, name, obj->path); 694 return -EIO; 695 } 696 697 *d_size = data->d_size; 698 return 0; 699 } 700 701 return -ENOENT; 702 } 703 704 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 705 __u32 *size) 706 { 707 int ret = -ENOENT; 708 size_t d_size; 709 710 *size = 0; 711 if (!name) { 712 return -EINVAL; 713 } else if (!strcmp(name, ".data")) { 714 if (obj->efile.data) 715 *size = obj->efile.data->d_size; 716 } else if (!strcmp(name, ".bss")) { 717 if (obj->efile.bss) 718 *size = obj->efile.bss->d_size; 719 } else if (!strcmp(name, ".rodata")) { 720 if (obj->efile.rodata) 721 *size = obj->efile.rodata->d_size; 722 } else { 723 ret = bpf_object_search_section_size(obj, name, &d_size); 724 if (!ret) 725 *size = d_size; 726 } 727 728 return *size ? 0 : ret; 729 } 730 731 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 732 __u32 *off) 733 { 734 Elf_Data *symbols = obj->efile.symbols; 735 const char *sname; 736 size_t si; 737 738 if (!name || !off) 739 return -EINVAL; 740 741 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 742 GElf_Sym sym; 743 744 if (!gelf_getsym(symbols, si, &sym)) 745 continue; 746 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 747 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 748 continue; 749 750 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 751 sym.st_name); 752 if (!sname) { 753 pr_warning("failed to get sym name string for var %s\n", 754 name); 755 return -EIO; 756 } 757 if (strcmp(name, sname) == 0) { 758 *off = sym.st_value; 759 return 0; 760 } 761 } 762 763 return -ENOENT; 764 } 765 766 static bool bpf_object__has_maps(const struct bpf_object *obj) 767 { 768 return obj->efile.maps_shndx >= 0 || 769 obj->efile.data_shndx >= 0 || 770 obj->efile.rodata_shndx >= 0 || 771 obj->efile.bss_shndx >= 0; 772 } 773 774 static int 775 bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map, 776 enum libbpf_map_type type, Elf_Data *data, 777 void **data_buff) 778 { 779 struct bpf_map_def *def = &map->def; 780 char map_name[BPF_OBJ_NAME_LEN]; 781 782 map->libbpf_type = type; 783 map->offset = ~(typeof(map->offset))0; 784 snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, 785 libbpf_type_to_btf_name[type]); 786 map->name = strdup(map_name); 787 if (!map->name) { 788 pr_warning("failed to alloc map name\n"); 789 return -ENOMEM; 790 } 791 792 def->type = BPF_MAP_TYPE_ARRAY; 793 def->key_size = sizeof(int); 794 def->value_size = data->d_size; 795 def->max_entries = 1; 796 def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0; 797 if (data_buff) { 798 *data_buff = malloc(data->d_size); 799 if (!*data_buff) { 800 zfree(&map->name); 801 pr_warning("failed to alloc map content buffer\n"); 802 return -ENOMEM; 803 } 804 memcpy(*data_buff, data->d_buf, data->d_size); 805 } 806 807 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 808 return 0; 809 } 810 811 static int bpf_object__init_maps(struct bpf_object *obj, int flags) 812 { 813 int i, map_idx, map_def_sz = 0, nr_syms, nr_maps = 0, nr_maps_glob = 0; 814 bool strict = !(flags & MAPS_RELAX_COMPAT); 815 Elf_Data *symbols = obj->efile.symbols; 816 Elf_Data *data = NULL; 817 int ret = 0; 818 819 if (!symbols) 820 return -EINVAL; 821 nr_syms = symbols->d_size / sizeof(GElf_Sym); 822 823 if (obj->efile.maps_shndx >= 0) { 824 Elf_Scn *scn = elf_getscn(obj->efile.elf, 825 obj->efile.maps_shndx); 826 827 if (scn) 828 data = elf_getdata(scn, NULL); 829 if (!scn || !data) { 830 pr_warning("failed to get Elf_Data from map section %d\n", 831 obj->efile.maps_shndx); 832 return -EINVAL; 833 } 834 } 835 836 /* 837 * Count number of maps. Each map has a name. 838 * Array of maps is not supported: only the first element is 839 * considered. 840 * 841 * TODO: Detect array of map and report error. 842 */ 843 if (obj->caps.global_data) { 844 if (obj->efile.data_shndx >= 0) 845 nr_maps_glob++; 846 if (obj->efile.rodata_shndx >= 0) 847 nr_maps_glob++; 848 if (obj->efile.bss_shndx >= 0) 849 nr_maps_glob++; 850 } 851 852 for (i = 0; data && i < nr_syms; i++) { 853 GElf_Sym sym; 854 855 if (!gelf_getsym(symbols, i, &sym)) 856 continue; 857 if (sym.st_shndx != obj->efile.maps_shndx) 858 continue; 859 nr_maps++; 860 } 861 862 if (!nr_maps && !nr_maps_glob) 863 return 0; 864 865 /* Assume equally sized map definitions */ 866 if (data) { 867 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 868 nr_maps, data->d_size); 869 870 map_def_sz = data->d_size / nr_maps; 871 if (!data->d_size || (data->d_size % nr_maps) != 0) { 872 pr_warning("unable to determine map definition size " 873 "section %s, %d maps in %zd bytes\n", 874 obj->path, nr_maps, data->d_size); 875 return -EINVAL; 876 } 877 } 878 879 nr_maps += nr_maps_glob; 880 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 881 if (!obj->maps) { 882 pr_warning("alloc maps for object failed\n"); 883 return -ENOMEM; 884 } 885 obj->nr_maps = nr_maps; 886 887 for (i = 0; i < nr_maps; i++) { 888 /* 889 * fill all fd with -1 so won't close incorrect 890 * fd (fd=0 is stdin) when failure (zclose won't close 891 * negative fd)). 892 */ 893 obj->maps[i].fd = -1; 894 obj->maps[i].inner_map_fd = -1; 895 } 896 897 /* 898 * Fill obj->maps using data in "maps" section. 899 */ 900 for (i = 0, map_idx = 0; data && i < nr_syms; i++) { 901 GElf_Sym sym; 902 const char *map_name; 903 struct bpf_map_def *def; 904 905 if (!gelf_getsym(symbols, i, &sym)) 906 continue; 907 if (sym.st_shndx != obj->efile.maps_shndx) 908 continue; 909 910 map_name = elf_strptr(obj->efile.elf, 911 obj->efile.strtabidx, 912 sym.st_name); 913 if (!map_name) { 914 pr_warning("failed to get map #%d name sym string for obj %s\n", 915 map_idx, obj->path); 916 return -LIBBPF_ERRNO__FORMAT; 917 } 918 919 obj->maps[map_idx].libbpf_type = LIBBPF_MAP_UNSPEC; 920 obj->maps[map_idx].offset = sym.st_value; 921 if (sym.st_value + map_def_sz > data->d_size) { 922 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 923 obj->path, map_name); 924 return -EINVAL; 925 } 926 927 obj->maps[map_idx].name = strdup(map_name); 928 if (!obj->maps[map_idx].name) { 929 pr_warning("failed to alloc map name\n"); 930 return -ENOMEM; 931 } 932 pr_debug("map %d is \"%s\"\n", map_idx, 933 obj->maps[map_idx].name); 934 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 935 /* 936 * If the definition of the map in the object file fits in 937 * bpf_map_def, copy it. Any extra fields in our version 938 * of bpf_map_def will default to zero as a result of the 939 * calloc above. 940 */ 941 if (map_def_sz <= sizeof(struct bpf_map_def)) { 942 memcpy(&obj->maps[map_idx].def, def, map_def_sz); 943 } else { 944 /* 945 * Here the map structure being read is bigger than what 946 * we expect, truncate if the excess bits are all zero. 947 * If they are not zero, reject this map as 948 * incompatible. 949 */ 950 char *b; 951 for (b = ((char *)def) + sizeof(struct bpf_map_def); 952 b < ((char *)def) + map_def_sz; b++) { 953 if (*b != 0) { 954 pr_warning("maps section in %s: \"%s\" " 955 "has unrecognized, non-zero " 956 "options\n", 957 obj->path, map_name); 958 if (strict) 959 return -EINVAL; 960 } 961 } 962 memcpy(&obj->maps[map_idx].def, def, 963 sizeof(struct bpf_map_def)); 964 } 965 map_idx++; 966 } 967 968 if (!obj->caps.global_data) 969 goto finalize; 970 971 /* 972 * Populate rest of obj->maps with libbpf internal maps. 973 */ 974 if (obj->efile.data_shndx >= 0) 975 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 976 LIBBPF_MAP_DATA, 977 obj->efile.data, 978 &obj->sections.data); 979 if (!ret && obj->efile.rodata_shndx >= 0) 980 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 981 LIBBPF_MAP_RODATA, 982 obj->efile.rodata, 983 &obj->sections.rodata); 984 if (!ret && obj->efile.bss_shndx >= 0) 985 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 986 LIBBPF_MAP_BSS, 987 obj->efile.bss, NULL); 988 finalize: 989 if (!ret) 990 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), 991 compare_bpf_map); 992 return ret; 993 } 994 995 static bool section_have_execinstr(struct bpf_object *obj, int idx) 996 { 997 Elf_Scn *scn; 998 GElf_Shdr sh; 999 1000 scn = elf_getscn(obj->efile.elf, idx); 1001 if (!scn) 1002 return false; 1003 1004 if (gelf_getshdr(scn, &sh) != &sh) 1005 return false; 1006 1007 if (sh.sh_flags & SHF_EXECINSTR) 1008 return true; 1009 1010 return false; 1011 } 1012 1013 static void bpf_object__sanitize_btf(struct bpf_object *obj) 1014 { 1015 bool has_datasec = obj->caps.btf_datasec; 1016 bool has_func = obj->caps.btf_func; 1017 struct btf *btf = obj->btf; 1018 struct btf_type *t; 1019 int i, j, vlen; 1020 __u16 kind; 1021 1022 if (!obj->btf || (has_func && has_datasec)) 1023 return; 1024 1025 for (i = 1; i <= btf__get_nr_types(btf); i++) { 1026 t = (struct btf_type *)btf__type_by_id(btf, i); 1027 kind = BTF_INFO_KIND(t->info); 1028 1029 if (!has_datasec && kind == BTF_KIND_VAR) { 1030 /* replace VAR with INT */ 1031 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 1032 t->size = sizeof(int); 1033 *(int *)(t+1) = BTF_INT_ENC(0, 0, 32); 1034 } else if (!has_datasec && kind == BTF_KIND_DATASEC) { 1035 /* replace DATASEC with STRUCT */ 1036 struct btf_var_secinfo *v = (void *)(t + 1); 1037 struct btf_member *m = (void *)(t + 1); 1038 struct btf_type *vt; 1039 char *name; 1040 1041 name = (char *)btf__name_by_offset(btf, t->name_off); 1042 while (*name) { 1043 if (*name == '.') 1044 *name = '_'; 1045 name++; 1046 } 1047 1048 vlen = BTF_INFO_VLEN(t->info); 1049 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 1050 for (j = 0; j < vlen; j++, v++, m++) { 1051 /* order of field assignments is important */ 1052 m->offset = v->offset * 8; 1053 m->type = v->type; 1054 /* preserve variable name as member name */ 1055 vt = (void *)btf__type_by_id(btf, v->type); 1056 m->name_off = vt->name_off; 1057 } 1058 } else if (!has_func && kind == BTF_KIND_FUNC_PROTO) { 1059 /* replace FUNC_PROTO with ENUM */ 1060 vlen = BTF_INFO_VLEN(t->info); 1061 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 1062 t->size = sizeof(__u32); /* kernel enforced */ 1063 } else if (!has_func && kind == BTF_KIND_FUNC) { 1064 /* replace FUNC with TYPEDEF */ 1065 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 1066 } 1067 } 1068 } 1069 1070 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) 1071 { 1072 if (!obj->btf_ext) 1073 return; 1074 1075 if (!obj->caps.btf_func) { 1076 btf_ext__free(obj->btf_ext); 1077 obj->btf_ext = NULL; 1078 } 1079 } 1080 1081 static int bpf_object__elf_collect(struct bpf_object *obj, int flags) 1082 { 1083 Elf *elf = obj->efile.elf; 1084 GElf_Ehdr *ep = &obj->efile.ehdr; 1085 Elf_Data *btf_ext_data = NULL; 1086 Elf_Data *btf_data = NULL; 1087 Elf_Scn *scn = NULL; 1088 int idx = 0, err = 0; 1089 1090 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1091 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 1092 pr_warning("failed to get e_shstrndx from %s\n", obj->path); 1093 return -LIBBPF_ERRNO__FORMAT; 1094 } 1095 1096 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1097 char *name; 1098 GElf_Shdr sh; 1099 Elf_Data *data; 1100 1101 idx++; 1102 if (gelf_getshdr(scn, &sh) != &sh) { 1103 pr_warning("failed to get section(%d) header from %s\n", 1104 idx, obj->path); 1105 err = -LIBBPF_ERRNO__FORMAT; 1106 goto out; 1107 } 1108 1109 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 1110 if (!name) { 1111 pr_warning("failed to get section(%d) name from %s\n", 1112 idx, obj->path); 1113 err = -LIBBPF_ERRNO__FORMAT; 1114 goto out; 1115 } 1116 1117 data = elf_getdata(scn, 0); 1118 if (!data) { 1119 pr_warning("failed to get section(%d) data from %s(%s)\n", 1120 idx, name, obj->path); 1121 err = -LIBBPF_ERRNO__FORMAT; 1122 goto out; 1123 } 1124 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 1125 idx, name, (unsigned long)data->d_size, 1126 (int)sh.sh_link, (unsigned long)sh.sh_flags, 1127 (int)sh.sh_type); 1128 1129 if (strcmp(name, "license") == 0) { 1130 err = bpf_object__init_license(obj, 1131 data->d_buf, 1132 data->d_size); 1133 } else if (strcmp(name, "version") == 0) { 1134 err = bpf_object__init_kversion(obj, 1135 data->d_buf, 1136 data->d_size); 1137 } else if (strcmp(name, "maps") == 0) { 1138 obj->efile.maps_shndx = idx; 1139 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 1140 btf_data = data; 1141 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 1142 btf_ext_data = data; 1143 } else if (sh.sh_type == SHT_SYMTAB) { 1144 if (obj->efile.symbols) { 1145 pr_warning("bpf: multiple SYMTAB in %s\n", 1146 obj->path); 1147 err = -LIBBPF_ERRNO__FORMAT; 1148 } else { 1149 obj->efile.symbols = data; 1150 obj->efile.strtabidx = sh.sh_link; 1151 } 1152 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 1153 if (sh.sh_flags & SHF_EXECINSTR) { 1154 if (strcmp(name, ".text") == 0) 1155 obj->efile.text_shndx = idx; 1156 err = bpf_object__add_program(obj, data->d_buf, 1157 data->d_size, name, idx); 1158 if (err) { 1159 char errmsg[STRERR_BUFSIZE]; 1160 char *cp = libbpf_strerror_r(-err, errmsg, 1161 sizeof(errmsg)); 1162 1163 pr_warning("failed to alloc program %s (%s): %s", 1164 name, obj->path, cp); 1165 } 1166 } else if (strcmp(name, ".data") == 0) { 1167 obj->efile.data = data; 1168 obj->efile.data_shndx = idx; 1169 } else if (strcmp(name, ".rodata") == 0) { 1170 obj->efile.rodata = data; 1171 obj->efile.rodata_shndx = idx; 1172 } else { 1173 pr_debug("skip section(%d) %s\n", idx, name); 1174 } 1175 } else if (sh.sh_type == SHT_REL) { 1176 void *reloc = obj->efile.reloc; 1177 int nr_reloc = obj->efile.nr_reloc + 1; 1178 int sec = sh.sh_info; /* points to other section */ 1179 1180 /* Only do relo for section with exec instructions */ 1181 if (!section_have_execinstr(obj, sec)) { 1182 pr_debug("skip relo %s(%d) for section(%d)\n", 1183 name, idx, sec); 1184 continue; 1185 } 1186 1187 reloc = reallocarray(reloc, nr_reloc, 1188 sizeof(*obj->efile.reloc)); 1189 if (!reloc) { 1190 pr_warning("realloc failed\n"); 1191 err = -ENOMEM; 1192 } else { 1193 int n = nr_reloc - 1; 1194 1195 obj->efile.reloc = reloc; 1196 obj->efile.nr_reloc = nr_reloc; 1197 1198 obj->efile.reloc[n].shdr = sh; 1199 obj->efile.reloc[n].data = data; 1200 } 1201 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { 1202 obj->efile.bss = data; 1203 obj->efile.bss_shndx = idx; 1204 } else { 1205 pr_debug("skip section(%d) %s\n", idx, name); 1206 } 1207 if (err) 1208 goto out; 1209 } 1210 1211 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 1212 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 1213 return -LIBBPF_ERRNO__FORMAT; 1214 } 1215 if (btf_data) { 1216 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 1217 if (IS_ERR(obj->btf)) { 1218 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1219 BTF_ELF_SEC, PTR_ERR(obj->btf)); 1220 obj->btf = NULL; 1221 } else { 1222 err = btf__finalize_data(obj, obj->btf); 1223 if (!err) { 1224 bpf_object__sanitize_btf(obj); 1225 err = btf__load(obj->btf); 1226 } 1227 if (err) { 1228 pr_warning("Error finalizing and loading %s into kernel: %d. Ignored and continue.\n", 1229 BTF_ELF_SEC, err); 1230 btf__free(obj->btf); 1231 obj->btf = NULL; 1232 err = 0; 1233 } 1234 } 1235 } 1236 if (btf_ext_data) { 1237 if (!obj->btf) { 1238 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 1239 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 1240 } else { 1241 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 1242 btf_ext_data->d_size); 1243 if (IS_ERR(obj->btf_ext)) { 1244 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1245 BTF_EXT_ELF_SEC, 1246 PTR_ERR(obj->btf_ext)); 1247 obj->btf_ext = NULL; 1248 } else { 1249 bpf_object__sanitize_btf_ext(obj); 1250 } 1251 } 1252 } 1253 if (bpf_object__has_maps(obj)) { 1254 err = bpf_object__init_maps(obj, flags); 1255 if (err) 1256 goto out; 1257 } 1258 err = bpf_object__init_prog_names(obj); 1259 out: 1260 return err; 1261 } 1262 1263 static struct bpf_program * 1264 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 1265 { 1266 struct bpf_program *prog; 1267 size_t i; 1268 1269 for (i = 0; i < obj->nr_programs; i++) { 1270 prog = &obj->programs[i]; 1271 if (prog->idx == idx) 1272 return prog; 1273 } 1274 return NULL; 1275 } 1276 1277 struct bpf_program * 1278 bpf_object__find_program_by_title(struct bpf_object *obj, const char *title) 1279 { 1280 struct bpf_program *pos; 1281 1282 bpf_object__for_each_program(pos, obj) { 1283 if (pos->section_name && !strcmp(pos->section_name, title)) 1284 return pos; 1285 } 1286 return NULL; 1287 } 1288 1289 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 1290 int shndx) 1291 { 1292 return shndx == obj->efile.data_shndx || 1293 shndx == obj->efile.bss_shndx || 1294 shndx == obj->efile.rodata_shndx; 1295 } 1296 1297 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 1298 int shndx) 1299 { 1300 return shndx == obj->efile.maps_shndx; 1301 } 1302 1303 static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, 1304 int shndx) 1305 { 1306 return shndx == obj->efile.text_shndx || 1307 bpf_object__shndx_is_maps(obj, shndx) || 1308 bpf_object__shndx_is_data(obj, shndx); 1309 } 1310 1311 static enum libbpf_map_type 1312 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 1313 { 1314 if (shndx == obj->efile.data_shndx) 1315 return LIBBPF_MAP_DATA; 1316 else if (shndx == obj->efile.bss_shndx) 1317 return LIBBPF_MAP_BSS; 1318 else if (shndx == obj->efile.rodata_shndx) 1319 return LIBBPF_MAP_RODATA; 1320 else 1321 return LIBBPF_MAP_UNSPEC; 1322 } 1323 1324 static int 1325 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 1326 Elf_Data *data, struct bpf_object *obj) 1327 { 1328 Elf_Data *symbols = obj->efile.symbols; 1329 struct bpf_map *maps = obj->maps; 1330 size_t nr_maps = obj->nr_maps; 1331 int i, nrels; 1332 1333 pr_debug("collecting relocating info for: '%s'\n", prog->section_name); 1334 nrels = shdr->sh_size / shdr->sh_entsize; 1335 1336 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 1337 if (!prog->reloc_desc) { 1338 pr_warning("failed to alloc memory in relocation\n"); 1339 return -ENOMEM; 1340 } 1341 prog->nr_reloc = nrels; 1342 1343 for (i = 0; i < nrels; i++) { 1344 GElf_Sym sym; 1345 GElf_Rel rel; 1346 unsigned int insn_idx; 1347 unsigned int shdr_idx; 1348 struct bpf_insn *insns = prog->insns; 1349 enum libbpf_map_type type; 1350 const char *name; 1351 size_t map_idx; 1352 1353 if (!gelf_getrel(data, i, &rel)) { 1354 pr_warning("relocation: failed to get %d reloc\n", i); 1355 return -LIBBPF_ERRNO__FORMAT; 1356 } 1357 1358 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 1359 pr_warning("relocation: symbol %"PRIx64" not found\n", 1360 GELF_R_SYM(rel.r_info)); 1361 return -LIBBPF_ERRNO__FORMAT; 1362 } 1363 1364 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1365 sym.st_name) ? : "<?>"; 1366 1367 pr_debug("relo for %lld value %lld name %d (\'%s\')\n", 1368 (long long) (rel.r_info >> 32), 1369 (long long) sym.st_value, sym.st_name, name); 1370 1371 shdr_idx = sym.st_shndx; 1372 if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { 1373 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", 1374 prog->section_name, shdr_idx); 1375 return -LIBBPF_ERRNO__RELOC; 1376 } 1377 1378 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 1379 pr_debug("relocation: insn_idx=%u\n", insn_idx); 1380 1381 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 1382 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 1383 pr_warning("incorrect bpf_call opcode\n"); 1384 return -LIBBPF_ERRNO__RELOC; 1385 } 1386 prog->reloc_desc[i].type = RELO_CALL; 1387 prog->reloc_desc[i].insn_idx = insn_idx; 1388 prog->reloc_desc[i].text_off = sym.st_value; 1389 obj->has_pseudo_calls = true; 1390 continue; 1391 } 1392 1393 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 1394 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 1395 insn_idx, insns[insn_idx].code); 1396 return -LIBBPF_ERRNO__RELOC; 1397 } 1398 1399 if (bpf_object__shndx_is_maps(obj, shdr_idx) || 1400 bpf_object__shndx_is_data(obj, shdr_idx)) { 1401 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 1402 if (type != LIBBPF_MAP_UNSPEC) { 1403 if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { 1404 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", 1405 name, insn_idx, insns[insn_idx].code); 1406 return -LIBBPF_ERRNO__RELOC; 1407 } 1408 if (!obj->caps.global_data) { 1409 pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", 1410 name, insn_idx); 1411 return -LIBBPF_ERRNO__RELOC; 1412 } 1413 } 1414 1415 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 1416 if (maps[map_idx].libbpf_type != type) 1417 continue; 1418 if (type != LIBBPF_MAP_UNSPEC || 1419 maps[map_idx].offset == sym.st_value) { 1420 pr_debug("relocation: find map %zd (%s) for insn %u\n", 1421 map_idx, maps[map_idx].name, insn_idx); 1422 break; 1423 } 1424 } 1425 1426 if (map_idx >= nr_maps) { 1427 pr_warning("bpf relocation: map_idx %d larger than %d\n", 1428 (int)map_idx, (int)nr_maps - 1); 1429 return -LIBBPF_ERRNO__RELOC; 1430 } 1431 1432 prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? 1433 RELO_DATA : RELO_LD64; 1434 prog->reloc_desc[i].insn_idx = insn_idx; 1435 prog->reloc_desc[i].map_idx = map_idx; 1436 } 1437 } 1438 return 0; 1439 } 1440 1441 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) 1442 { 1443 struct bpf_map_def *def = &map->def; 1444 __u32 key_type_id = 0, value_type_id = 0; 1445 int ret; 1446 1447 if (!bpf_map__is_internal(map)) { 1448 ret = btf__get_map_kv_tids(btf, map->name, def->key_size, 1449 def->value_size, &key_type_id, 1450 &value_type_id); 1451 } else { 1452 /* 1453 * LLVM annotates global data differently in BTF, that is, 1454 * only as '.data', '.bss' or '.rodata'. 1455 */ 1456 ret = btf__find_by_name(btf, 1457 libbpf_type_to_btf_name[map->libbpf_type]); 1458 } 1459 if (ret < 0) 1460 return ret; 1461 1462 map->btf_key_type_id = key_type_id; 1463 map->btf_value_type_id = bpf_map__is_internal(map) ? 1464 ret : value_type_id; 1465 return 0; 1466 } 1467 1468 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 1469 { 1470 struct bpf_map_info info = {}; 1471 __u32 len = sizeof(info); 1472 int new_fd, err; 1473 char *new_name; 1474 1475 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1476 if (err) 1477 return err; 1478 1479 new_name = strdup(info.name); 1480 if (!new_name) 1481 return -errno; 1482 1483 new_fd = open("/", O_RDONLY | O_CLOEXEC); 1484 if (new_fd < 0) 1485 goto err_free_new_name; 1486 1487 new_fd = dup3(fd, new_fd, O_CLOEXEC); 1488 if (new_fd < 0) 1489 goto err_close_new_fd; 1490 1491 err = zclose(map->fd); 1492 if (err) 1493 goto err_close_new_fd; 1494 free(map->name); 1495 1496 map->fd = new_fd; 1497 map->name = new_name; 1498 map->def.type = info.type; 1499 map->def.key_size = info.key_size; 1500 map->def.value_size = info.value_size; 1501 map->def.max_entries = info.max_entries; 1502 map->def.map_flags = info.map_flags; 1503 map->btf_key_type_id = info.btf_key_type_id; 1504 map->btf_value_type_id = info.btf_value_type_id; 1505 1506 return 0; 1507 1508 err_close_new_fd: 1509 close(new_fd); 1510 err_free_new_name: 1511 free(new_name); 1512 return -errno; 1513 } 1514 1515 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 1516 { 1517 if (!map || !max_entries) 1518 return -EINVAL; 1519 1520 /* If map already created, its attributes can't be changed. */ 1521 if (map->fd >= 0) 1522 return -EBUSY; 1523 1524 map->def.max_entries = max_entries; 1525 1526 return 0; 1527 } 1528 1529 static int 1530 bpf_object__probe_name(struct bpf_object *obj) 1531 { 1532 struct bpf_load_program_attr attr; 1533 char *cp, errmsg[STRERR_BUFSIZE]; 1534 struct bpf_insn insns[] = { 1535 BPF_MOV64_IMM(BPF_REG_0, 0), 1536 BPF_EXIT_INSN(), 1537 }; 1538 int ret; 1539 1540 /* make sure basic loading works */ 1541 1542 memset(&attr, 0, sizeof(attr)); 1543 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1544 attr.insns = insns; 1545 attr.insns_cnt = ARRAY_SIZE(insns); 1546 attr.license = "GPL"; 1547 1548 ret = bpf_load_program_xattr(&attr, NULL, 0); 1549 if (ret < 0) { 1550 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1551 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", 1552 __func__, cp, errno); 1553 return -errno; 1554 } 1555 close(ret); 1556 1557 /* now try the same program, but with the name */ 1558 1559 attr.name = "test"; 1560 ret = bpf_load_program_xattr(&attr, NULL, 0); 1561 if (ret >= 0) { 1562 obj->caps.name = 1; 1563 close(ret); 1564 } 1565 1566 return 0; 1567 } 1568 1569 static int 1570 bpf_object__probe_global_data(struct bpf_object *obj) 1571 { 1572 struct bpf_load_program_attr prg_attr; 1573 struct bpf_create_map_attr map_attr; 1574 char *cp, errmsg[STRERR_BUFSIZE]; 1575 struct bpf_insn insns[] = { 1576 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 1577 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 1578 BPF_MOV64_IMM(BPF_REG_0, 0), 1579 BPF_EXIT_INSN(), 1580 }; 1581 int ret, map; 1582 1583 memset(&map_attr, 0, sizeof(map_attr)); 1584 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 1585 map_attr.key_size = sizeof(int); 1586 map_attr.value_size = 32; 1587 map_attr.max_entries = 1; 1588 1589 map = bpf_create_map_xattr(&map_attr); 1590 if (map < 0) { 1591 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1592 pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", 1593 __func__, cp, errno); 1594 return -errno; 1595 } 1596 1597 insns[0].imm = map; 1598 1599 memset(&prg_attr, 0, sizeof(prg_attr)); 1600 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1601 prg_attr.insns = insns; 1602 prg_attr.insns_cnt = ARRAY_SIZE(insns); 1603 prg_attr.license = "GPL"; 1604 1605 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 1606 if (ret >= 0) { 1607 obj->caps.global_data = 1; 1608 close(ret); 1609 } 1610 1611 close(map); 1612 return 0; 1613 } 1614 1615 static int bpf_object__probe_btf_func(struct bpf_object *obj) 1616 { 1617 const char strs[] = "\0int\0x\0a"; 1618 /* void x(int a) {} */ 1619 __u32 types[] = { 1620 /* int */ 1621 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 1622 /* FUNC_PROTO */ /* [2] */ 1623 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 1624 BTF_PARAM_ENC(7, 1), 1625 /* FUNC x */ /* [3] */ 1626 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 1627 }; 1628 int btf_fd; 1629 1630 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 1631 strs, sizeof(strs)); 1632 if (btf_fd >= 0) { 1633 obj->caps.btf_func = 1; 1634 close(btf_fd); 1635 return 1; 1636 } 1637 1638 return 0; 1639 } 1640 1641 static int bpf_object__probe_btf_datasec(struct bpf_object *obj) 1642 { 1643 const char strs[] = "\0x\0.data"; 1644 /* static int a; */ 1645 __u32 types[] = { 1646 /* int */ 1647 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 1648 /* VAR x */ /* [2] */ 1649 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 1650 BTF_VAR_STATIC, 1651 /* DATASEC val */ /* [3] */ 1652 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 1653 BTF_VAR_SECINFO_ENC(2, 0, 4), 1654 }; 1655 int btf_fd; 1656 1657 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), 1658 strs, sizeof(strs)); 1659 if (btf_fd >= 0) { 1660 obj->caps.btf_datasec = 1; 1661 close(btf_fd); 1662 return 1; 1663 } 1664 1665 return 0; 1666 } 1667 1668 static int 1669 bpf_object__probe_caps(struct bpf_object *obj) 1670 { 1671 int (*probe_fn[])(struct bpf_object *obj) = { 1672 bpf_object__probe_name, 1673 bpf_object__probe_global_data, 1674 bpf_object__probe_btf_func, 1675 bpf_object__probe_btf_datasec, 1676 }; 1677 int i, ret; 1678 1679 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { 1680 ret = probe_fn[i](obj); 1681 if (ret < 0) 1682 pr_debug("Probe #%d failed with %d.\n", i, ret); 1683 } 1684 1685 return 0; 1686 } 1687 1688 static int 1689 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 1690 { 1691 char *cp, errmsg[STRERR_BUFSIZE]; 1692 int err, zero = 0; 1693 __u8 *data; 1694 1695 /* Nothing to do here since kernel already zero-initializes .bss map. */ 1696 if (map->libbpf_type == LIBBPF_MAP_BSS) 1697 return 0; 1698 1699 data = map->libbpf_type == LIBBPF_MAP_DATA ? 1700 obj->sections.data : obj->sections.rodata; 1701 1702 err = bpf_map_update_elem(map->fd, &zero, data, 0); 1703 /* Freeze .rodata map as read-only from syscall side. */ 1704 if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { 1705 err = bpf_map_freeze(map->fd); 1706 if (err) { 1707 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1708 pr_warning("Error freezing map(%s) as read-only: %s\n", 1709 map->name, cp); 1710 err = 0; 1711 } 1712 } 1713 return err; 1714 } 1715 1716 static int 1717 bpf_object__create_maps(struct bpf_object *obj) 1718 { 1719 struct bpf_create_map_attr create_attr = {}; 1720 unsigned int i; 1721 int err; 1722 1723 for (i = 0; i < obj->nr_maps; i++) { 1724 struct bpf_map *map = &obj->maps[i]; 1725 struct bpf_map_def *def = &map->def; 1726 char *cp, errmsg[STRERR_BUFSIZE]; 1727 int *pfd = &map->fd; 1728 1729 if (map->fd >= 0) { 1730 pr_debug("skip map create (preset) %s: fd=%d\n", 1731 map->name, map->fd); 1732 continue; 1733 } 1734 1735 if (obj->caps.name) 1736 create_attr.name = map->name; 1737 create_attr.map_ifindex = map->map_ifindex; 1738 create_attr.map_type = def->type; 1739 create_attr.map_flags = def->map_flags; 1740 create_attr.key_size = def->key_size; 1741 create_attr.value_size = def->value_size; 1742 create_attr.max_entries = def->max_entries; 1743 create_attr.btf_fd = -1; 1744 create_attr.btf_key_type_id = 0; 1745 create_attr.btf_value_type_id = 0; 1746 if (bpf_map_type__is_map_in_map(def->type) && 1747 map->inner_map_fd >= 0) 1748 create_attr.inner_map_fd = map->inner_map_fd; 1749 1750 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) { 1751 create_attr.btf_fd = btf__fd(obj->btf); 1752 create_attr.btf_key_type_id = map->btf_key_type_id; 1753 create_attr.btf_value_type_id = map->btf_value_type_id; 1754 } 1755 1756 *pfd = bpf_create_map_xattr(&create_attr); 1757 if (*pfd < 0 && create_attr.btf_fd >= 0) { 1758 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1759 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 1760 map->name, cp, errno); 1761 create_attr.btf_fd = -1; 1762 create_attr.btf_key_type_id = 0; 1763 create_attr.btf_value_type_id = 0; 1764 map->btf_key_type_id = 0; 1765 map->btf_value_type_id = 0; 1766 *pfd = bpf_create_map_xattr(&create_attr); 1767 } 1768 1769 if (*pfd < 0) { 1770 size_t j; 1771 1772 err = *pfd; 1773 err_out: 1774 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1775 pr_warning("failed to create map (name: '%s'): %s\n", 1776 map->name, cp); 1777 for (j = 0; j < i; j++) 1778 zclose(obj->maps[j].fd); 1779 return err; 1780 } 1781 1782 if (bpf_map__is_internal(map)) { 1783 err = bpf_object__populate_internal_map(obj, map); 1784 if (err < 0) { 1785 zclose(*pfd); 1786 goto err_out; 1787 } 1788 } 1789 1790 pr_debug("created map %s: fd=%d\n", map->name, *pfd); 1791 } 1792 1793 return 0; 1794 } 1795 1796 static int 1797 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 1798 void *btf_prog_info, const char *info_name) 1799 { 1800 if (err != -ENOENT) { 1801 pr_warning("Error in loading %s for sec %s.\n", 1802 info_name, prog->section_name); 1803 return err; 1804 } 1805 1806 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 1807 1808 if (btf_prog_info) { 1809 /* 1810 * Some info has already been found but has problem 1811 * in the last btf_ext reloc. Must have to error out. 1812 */ 1813 pr_warning("Error in relocating %s for sec %s.\n", 1814 info_name, prog->section_name); 1815 return err; 1816 } 1817 1818 /* Have problem loading the very first info. Ignore the rest. */ 1819 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", 1820 info_name, prog->section_name, info_name); 1821 return 0; 1822 } 1823 1824 static int 1825 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 1826 const char *section_name, __u32 insn_offset) 1827 { 1828 int err; 1829 1830 if (!insn_offset || prog->func_info) { 1831 /* 1832 * !insn_offset => main program 1833 * 1834 * For sub prog, the main program's func_info has to 1835 * be loaded first (i.e. prog->func_info != NULL) 1836 */ 1837 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 1838 section_name, insn_offset, 1839 &prog->func_info, 1840 &prog->func_info_cnt); 1841 if (err) 1842 return check_btf_ext_reloc_err(prog, err, 1843 prog->func_info, 1844 "bpf_func_info"); 1845 1846 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 1847 } 1848 1849 if (!insn_offset || prog->line_info) { 1850 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 1851 section_name, insn_offset, 1852 &prog->line_info, 1853 &prog->line_info_cnt); 1854 if (err) 1855 return check_btf_ext_reloc_err(prog, err, 1856 prog->line_info, 1857 "bpf_line_info"); 1858 1859 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 1860 } 1861 1862 if (!insn_offset) 1863 prog->btf_fd = btf__fd(obj->btf); 1864 1865 return 0; 1866 } 1867 1868 static int 1869 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 1870 struct reloc_desc *relo) 1871 { 1872 struct bpf_insn *insn, *new_insn; 1873 struct bpf_program *text; 1874 size_t new_cnt; 1875 int err; 1876 1877 if (relo->type != RELO_CALL) 1878 return -LIBBPF_ERRNO__RELOC; 1879 1880 if (prog->idx == obj->efile.text_shndx) { 1881 pr_warning("relo in .text insn %d into off %d\n", 1882 relo->insn_idx, relo->text_off); 1883 return -LIBBPF_ERRNO__RELOC; 1884 } 1885 1886 if (prog->main_prog_cnt == 0) { 1887 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 1888 if (!text) { 1889 pr_warning("no .text section found yet relo into text exist\n"); 1890 return -LIBBPF_ERRNO__RELOC; 1891 } 1892 new_cnt = prog->insns_cnt + text->insns_cnt; 1893 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 1894 if (!new_insn) { 1895 pr_warning("oom in prog realloc\n"); 1896 return -ENOMEM; 1897 } 1898 1899 if (obj->btf_ext) { 1900 err = bpf_program_reloc_btf_ext(prog, obj, 1901 text->section_name, 1902 prog->insns_cnt); 1903 if (err) 1904 return err; 1905 } 1906 1907 memcpy(new_insn + prog->insns_cnt, text->insns, 1908 text->insns_cnt * sizeof(*insn)); 1909 prog->insns = new_insn; 1910 prog->main_prog_cnt = prog->insns_cnt; 1911 prog->insns_cnt = new_cnt; 1912 pr_debug("added %zd insn from %s to prog %s\n", 1913 text->insns_cnt, text->section_name, 1914 prog->section_name); 1915 } 1916 insn = &prog->insns[relo->insn_idx]; 1917 insn->imm += prog->main_prog_cnt - relo->insn_idx; 1918 return 0; 1919 } 1920 1921 static int 1922 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1923 { 1924 int i, err; 1925 1926 if (!prog) 1927 return 0; 1928 1929 if (obj->btf_ext) { 1930 err = bpf_program_reloc_btf_ext(prog, obj, 1931 prog->section_name, 0); 1932 if (err) 1933 return err; 1934 } 1935 1936 if (!prog->reloc_desc) 1937 return 0; 1938 1939 for (i = 0; i < prog->nr_reloc; i++) { 1940 if (prog->reloc_desc[i].type == RELO_LD64 || 1941 prog->reloc_desc[i].type == RELO_DATA) { 1942 bool relo_data = prog->reloc_desc[i].type == RELO_DATA; 1943 struct bpf_insn *insns = prog->insns; 1944 int insn_idx, map_idx; 1945 1946 insn_idx = prog->reloc_desc[i].insn_idx; 1947 map_idx = prog->reloc_desc[i].map_idx; 1948 1949 if (insn_idx + 1 >= (int)prog->insns_cnt) { 1950 pr_warning("relocation out of range: '%s'\n", 1951 prog->section_name); 1952 return -LIBBPF_ERRNO__RELOC; 1953 } 1954 1955 if (!relo_data) { 1956 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1957 } else { 1958 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; 1959 insns[insn_idx + 1].imm = insns[insn_idx].imm; 1960 } 1961 insns[insn_idx].imm = obj->maps[map_idx].fd; 1962 } else if (prog->reloc_desc[i].type == RELO_CALL) { 1963 err = bpf_program__reloc_text(prog, obj, 1964 &prog->reloc_desc[i]); 1965 if (err) 1966 return err; 1967 } 1968 } 1969 1970 zfree(&prog->reloc_desc); 1971 prog->nr_reloc = 0; 1972 return 0; 1973 } 1974 1975 1976 static int 1977 bpf_object__relocate(struct bpf_object *obj) 1978 { 1979 struct bpf_program *prog; 1980 size_t i; 1981 int err; 1982 1983 for (i = 0; i < obj->nr_programs; i++) { 1984 prog = &obj->programs[i]; 1985 1986 err = bpf_program__relocate(prog, obj); 1987 if (err) { 1988 pr_warning("failed to relocate '%s'\n", 1989 prog->section_name); 1990 return err; 1991 } 1992 } 1993 return 0; 1994 } 1995 1996 static int bpf_object__collect_reloc(struct bpf_object *obj) 1997 { 1998 int i, err; 1999 2000 if (!obj_elf_valid(obj)) { 2001 pr_warning("Internal error: elf object is closed\n"); 2002 return -LIBBPF_ERRNO__INTERNAL; 2003 } 2004 2005 for (i = 0; i < obj->efile.nr_reloc; i++) { 2006 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 2007 Elf_Data *data = obj->efile.reloc[i].data; 2008 int idx = shdr->sh_info; 2009 struct bpf_program *prog; 2010 2011 if (shdr->sh_type != SHT_REL) { 2012 pr_warning("internal error at %d\n", __LINE__); 2013 return -LIBBPF_ERRNO__INTERNAL; 2014 } 2015 2016 prog = bpf_object__find_prog_by_idx(obj, idx); 2017 if (!prog) { 2018 pr_warning("relocation failed: no section(%d)\n", idx); 2019 return -LIBBPF_ERRNO__RELOC; 2020 } 2021 2022 err = bpf_program__collect_reloc(prog, shdr, data, obj); 2023 if (err) 2024 return err; 2025 } 2026 return 0; 2027 } 2028 2029 static int 2030 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 2031 char *license, __u32 kern_version, int *pfd) 2032 { 2033 struct bpf_load_program_attr load_attr; 2034 char *cp, errmsg[STRERR_BUFSIZE]; 2035 int log_buf_size = BPF_LOG_BUF_SIZE; 2036 char *log_buf; 2037 int ret; 2038 2039 if (!insns || !insns_cnt) 2040 return -EINVAL; 2041 2042 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 2043 load_attr.prog_type = prog->type; 2044 load_attr.expected_attach_type = prog->expected_attach_type; 2045 if (prog->caps->name) 2046 load_attr.name = prog->name; 2047 load_attr.insns = insns; 2048 load_attr.insns_cnt = insns_cnt; 2049 load_attr.license = license; 2050 load_attr.kern_version = kern_version; 2051 load_attr.prog_ifindex = prog->prog_ifindex; 2052 load_attr.prog_btf_fd = prog->btf_fd; 2053 load_attr.func_info = prog->func_info; 2054 load_attr.func_info_rec_size = prog->func_info_rec_size; 2055 load_attr.func_info_cnt = prog->func_info_cnt; 2056 load_attr.line_info = prog->line_info; 2057 load_attr.line_info_rec_size = prog->line_info_rec_size; 2058 load_attr.line_info_cnt = prog->line_info_cnt; 2059 load_attr.log_level = prog->log_level; 2060 load_attr.prog_flags = prog->prog_flags; 2061 2062 retry_load: 2063 log_buf = malloc(log_buf_size); 2064 if (!log_buf) 2065 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 2066 2067 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); 2068 2069 if (ret >= 0) { 2070 if (load_attr.log_level) 2071 pr_debug("verifier log:\n%s", log_buf); 2072 *pfd = ret; 2073 ret = 0; 2074 goto out; 2075 } 2076 2077 if (errno == ENOSPC) { 2078 log_buf_size <<= 1; 2079 free(log_buf); 2080 goto retry_load; 2081 } 2082 ret = -LIBBPF_ERRNO__LOAD; 2083 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2084 pr_warning("load bpf program failed: %s\n", cp); 2085 2086 if (log_buf && log_buf[0] != '\0') { 2087 ret = -LIBBPF_ERRNO__VERIFY; 2088 pr_warning("-- BEGIN DUMP LOG ---\n"); 2089 pr_warning("\n%s\n", log_buf); 2090 pr_warning("-- END LOG --\n"); 2091 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 2092 pr_warning("Program too large (%zu insns), at most %d insns\n", 2093 load_attr.insns_cnt, BPF_MAXINSNS); 2094 ret = -LIBBPF_ERRNO__PROG2BIG; 2095 } else { 2096 /* Wrong program type? */ 2097 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 2098 int fd; 2099 2100 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 2101 load_attr.expected_attach_type = 0; 2102 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 2103 if (fd >= 0) { 2104 close(fd); 2105 ret = -LIBBPF_ERRNO__PROGTYPE; 2106 goto out; 2107 } 2108 } 2109 2110 if (log_buf) 2111 ret = -LIBBPF_ERRNO__KVER; 2112 } 2113 2114 out: 2115 free(log_buf); 2116 return ret; 2117 } 2118 2119 int 2120 bpf_program__load(struct bpf_program *prog, 2121 char *license, __u32 kern_version) 2122 { 2123 int err = 0, fd, i; 2124 2125 if (prog->instances.nr < 0 || !prog->instances.fds) { 2126 if (prog->preprocessor) { 2127 pr_warning("Internal error: can't load program '%s'\n", 2128 prog->section_name); 2129 return -LIBBPF_ERRNO__INTERNAL; 2130 } 2131 2132 prog->instances.fds = malloc(sizeof(int)); 2133 if (!prog->instances.fds) { 2134 pr_warning("Not enough memory for BPF fds\n"); 2135 return -ENOMEM; 2136 } 2137 prog->instances.nr = 1; 2138 prog->instances.fds[0] = -1; 2139 } 2140 2141 if (!prog->preprocessor) { 2142 if (prog->instances.nr != 1) { 2143 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 2144 prog->section_name, prog->instances.nr); 2145 } 2146 err = load_program(prog, prog->insns, prog->insns_cnt, 2147 license, kern_version, &fd); 2148 if (!err) 2149 prog->instances.fds[0] = fd; 2150 goto out; 2151 } 2152 2153 for (i = 0; i < prog->instances.nr; i++) { 2154 struct bpf_prog_prep_result result; 2155 bpf_program_prep_t preprocessor = prog->preprocessor; 2156 2157 memset(&result, 0, sizeof(result)); 2158 err = preprocessor(prog, i, prog->insns, 2159 prog->insns_cnt, &result); 2160 if (err) { 2161 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 2162 i, prog->section_name); 2163 goto out; 2164 } 2165 2166 if (!result.new_insn_ptr || !result.new_insn_cnt) { 2167 pr_debug("Skip loading the %dth instance of program '%s'\n", 2168 i, prog->section_name); 2169 prog->instances.fds[i] = -1; 2170 if (result.pfd) 2171 *result.pfd = -1; 2172 continue; 2173 } 2174 2175 err = load_program(prog, result.new_insn_ptr, 2176 result.new_insn_cnt, 2177 license, kern_version, &fd); 2178 2179 if (err) { 2180 pr_warning("Loading the %dth instance of program '%s' failed\n", 2181 i, prog->section_name); 2182 goto out; 2183 } 2184 2185 if (result.pfd) 2186 *result.pfd = fd; 2187 prog->instances.fds[i] = fd; 2188 } 2189 out: 2190 if (err) 2191 pr_warning("failed to load program '%s'\n", 2192 prog->section_name); 2193 zfree(&prog->insns); 2194 prog->insns_cnt = 0; 2195 return err; 2196 } 2197 2198 static bool bpf_program__is_function_storage(struct bpf_program *prog, 2199 struct bpf_object *obj) 2200 { 2201 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 2202 } 2203 2204 static int 2205 bpf_object__load_progs(struct bpf_object *obj, int log_level) 2206 { 2207 size_t i; 2208 int err; 2209 2210 for (i = 0; i < obj->nr_programs; i++) { 2211 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 2212 continue; 2213 obj->programs[i].log_level |= log_level; 2214 err = bpf_program__load(&obj->programs[i], 2215 obj->license, 2216 obj->kern_version); 2217 if (err) 2218 return err; 2219 } 2220 return 0; 2221 } 2222 2223 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) 2224 { 2225 switch (type) { 2226 case BPF_PROG_TYPE_SOCKET_FILTER: 2227 case BPF_PROG_TYPE_SCHED_CLS: 2228 case BPF_PROG_TYPE_SCHED_ACT: 2229 case BPF_PROG_TYPE_XDP: 2230 case BPF_PROG_TYPE_CGROUP_SKB: 2231 case BPF_PROG_TYPE_CGROUP_SOCK: 2232 case BPF_PROG_TYPE_LWT_IN: 2233 case BPF_PROG_TYPE_LWT_OUT: 2234 case BPF_PROG_TYPE_LWT_XMIT: 2235 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 2236 case BPF_PROG_TYPE_SOCK_OPS: 2237 case BPF_PROG_TYPE_SK_SKB: 2238 case BPF_PROG_TYPE_CGROUP_DEVICE: 2239 case BPF_PROG_TYPE_SK_MSG: 2240 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2241 case BPF_PROG_TYPE_LIRC_MODE2: 2242 case BPF_PROG_TYPE_SK_REUSEPORT: 2243 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2244 case BPF_PROG_TYPE_UNSPEC: 2245 case BPF_PROG_TYPE_TRACEPOINT: 2246 case BPF_PROG_TYPE_RAW_TRACEPOINT: 2247 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2248 case BPF_PROG_TYPE_PERF_EVENT: 2249 case BPF_PROG_TYPE_CGROUP_SYSCTL: 2250 return false; 2251 case BPF_PROG_TYPE_KPROBE: 2252 default: 2253 return true; 2254 } 2255 } 2256 2257 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) 2258 { 2259 if (needs_kver && obj->kern_version == 0) { 2260 pr_warning("%s doesn't provide kernel version\n", 2261 obj->path); 2262 return -LIBBPF_ERRNO__KVERSION; 2263 } 2264 return 0; 2265 } 2266 2267 static struct bpf_object * 2268 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, 2269 bool needs_kver, int flags) 2270 { 2271 struct bpf_object *obj; 2272 int err; 2273 2274 if (elf_version(EV_CURRENT) == EV_NONE) { 2275 pr_warning("failed to init libelf for %s\n", path); 2276 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 2277 } 2278 2279 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 2280 if (IS_ERR(obj)) 2281 return obj; 2282 2283 CHECK_ERR(bpf_object__elf_init(obj), err, out); 2284 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 2285 CHECK_ERR(bpf_object__probe_caps(obj), err, out); 2286 CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); 2287 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 2288 CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); 2289 2290 bpf_object__elf_finish(obj); 2291 return obj; 2292 out: 2293 bpf_object__close(obj); 2294 return ERR_PTR(err); 2295 } 2296 2297 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, 2298 int flags) 2299 { 2300 /* param validation */ 2301 if (!attr->file) 2302 return NULL; 2303 2304 pr_debug("loading %s\n", attr->file); 2305 2306 return __bpf_object__open(attr->file, NULL, 0, 2307 bpf_prog_type__needs_kver(attr->prog_type), 2308 flags); 2309 } 2310 2311 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 2312 { 2313 return __bpf_object__open_xattr(attr, 0); 2314 } 2315 2316 struct bpf_object *bpf_object__open(const char *path) 2317 { 2318 struct bpf_object_open_attr attr = { 2319 .file = path, 2320 .prog_type = BPF_PROG_TYPE_UNSPEC, 2321 }; 2322 2323 return bpf_object__open_xattr(&attr); 2324 } 2325 2326 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 2327 size_t obj_buf_sz, 2328 const char *name) 2329 { 2330 char tmp_name[64]; 2331 2332 /* param validation */ 2333 if (!obj_buf || obj_buf_sz <= 0) 2334 return NULL; 2335 2336 if (!name) { 2337 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 2338 (unsigned long)obj_buf, 2339 (unsigned long)obj_buf_sz); 2340 name = tmp_name; 2341 } 2342 pr_debug("loading object '%s' from buffer\n", name); 2343 2344 return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); 2345 } 2346 2347 int bpf_object__unload(struct bpf_object *obj) 2348 { 2349 size_t i; 2350 2351 if (!obj) 2352 return -EINVAL; 2353 2354 for (i = 0; i < obj->nr_maps; i++) 2355 zclose(obj->maps[i].fd); 2356 2357 for (i = 0; i < obj->nr_programs; i++) 2358 bpf_program__unload(&obj->programs[i]); 2359 2360 return 0; 2361 } 2362 2363 int bpf_object__load_xattr(struct bpf_object_load_attr *attr) 2364 { 2365 struct bpf_object *obj; 2366 int err; 2367 2368 if (!attr) 2369 return -EINVAL; 2370 obj = attr->obj; 2371 if (!obj) 2372 return -EINVAL; 2373 2374 if (obj->loaded) { 2375 pr_warning("object should not be loaded twice\n"); 2376 return -EINVAL; 2377 } 2378 2379 obj->loaded = true; 2380 2381 CHECK_ERR(bpf_object__create_maps(obj), err, out); 2382 CHECK_ERR(bpf_object__relocate(obj), err, out); 2383 CHECK_ERR(bpf_object__load_progs(obj, attr->log_level), err, out); 2384 2385 return 0; 2386 out: 2387 bpf_object__unload(obj); 2388 pr_warning("failed to load object '%s'\n", obj->path); 2389 return err; 2390 } 2391 2392 int bpf_object__load(struct bpf_object *obj) 2393 { 2394 struct bpf_object_load_attr attr = { 2395 .obj = obj, 2396 }; 2397 2398 return bpf_object__load_xattr(&attr); 2399 } 2400 2401 static int check_path(const char *path) 2402 { 2403 char *cp, errmsg[STRERR_BUFSIZE]; 2404 struct statfs st_fs; 2405 char *dname, *dir; 2406 int err = 0; 2407 2408 if (path == NULL) 2409 return -EINVAL; 2410 2411 dname = strdup(path); 2412 if (dname == NULL) 2413 return -ENOMEM; 2414 2415 dir = dirname(dname); 2416 if (statfs(dir, &st_fs)) { 2417 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2418 pr_warning("failed to statfs %s: %s\n", dir, cp); 2419 err = -errno; 2420 } 2421 free(dname); 2422 2423 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 2424 pr_warning("specified path %s is not on BPF FS\n", path); 2425 err = -EINVAL; 2426 } 2427 2428 return err; 2429 } 2430 2431 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 2432 int instance) 2433 { 2434 char *cp, errmsg[STRERR_BUFSIZE]; 2435 int err; 2436 2437 err = check_path(path); 2438 if (err) 2439 return err; 2440 2441 if (prog == NULL) { 2442 pr_warning("invalid program pointer\n"); 2443 return -EINVAL; 2444 } 2445 2446 if (instance < 0 || instance >= prog->instances.nr) { 2447 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2448 instance, prog->section_name, prog->instances.nr); 2449 return -EINVAL; 2450 } 2451 2452 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 2453 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2454 pr_warning("failed to pin program: %s\n", cp); 2455 return -errno; 2456 } 2457 pr_debug("pinned program '%s'\n", path); 2458 2459 return 0; 2460 } 2461 2462 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 2463 int instance) 2464 { 2465 int err; 2466 2467 err = check_path(path); 2468 if (err) 2469 return err; 2470 2471 if (prog == NULL) { 2472 pr_warning("invalid program pointer\n"); 2473 return -EINVAL; 2474 } 2475 2476 if (instance < 0 || instance >= prog->instances.nr) { 2477 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2478 instance, prog->section_name, prog->instances.nr); 2479 return -EINVAL; 2480 } 2481 2482 err = unlink(path); 2483 if (err != 0) 2484 return -errno; 2485 pr_debug("unpinned program '%s'\n", path); 2486 2487 return 0; 2488 } 2489 2490 static int make_dir(const char *path) 2491 { 2492 char *cp, errmsg[STRERR_BUFSIZE]; 2493 int err = 0; 2494 2495 if (mkdir(path, 0700) && errno != EEXIST) 2496 err = -errno; 2497 2498 if (err) { 2499 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 2500 pr_warning("failed to mkdir %s: %s\n", path, cp); 2501 } 2502 return err; 2503 } 2504 2505 int bpf_program__pin(struct bpf_program *prog, const char *path) 2506 { 2507 int i, err; 2508 2509 err = check_path(path); 2510 if (err) 2511 return err; 2512 2513 if (prog == NULL) { 2514 pr_warning("invalid program pointer\n"); 2515 return -EINVAL; 2516 } 2517 2518 if (prog->instances.nr <= 0) { 2519 pr_warning("no instances of prog %s to pin\n", 2520 prog->section_name); 2521 return -EINVAL; 2522 } 2523 2524 if (prog->instances.nr == 1) { 2525 /* don't create subdirs when pinning single instance */ 2526 return bpf_program__pin_instance(prog, path, 0); 2527 } 2528 2529 err = make_dir(path); 2530 if (err) 2531 return err; 2532 2533 for (i = 0; i < prog->instances.nr; i++) { 2534 char buf[PATH_MAX]; 2535 int len; 2536 2537 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2538 if (len < 0) { 2539 err = -EINVAL; 2540 goto err_unpin; 2541 } else if (len >= PATH_MAX) { 2542 err = -ENAMETOOLONG; 2543 goto err_unpin; 2544 } 2545 2546 err = bpf_program__pin_instance(prog, buf, i); 2547 if (err) 2548 goto err_unpin; 2549 } 2550 2551 return 0; 2552 2553 err_unpin: 2554 for (i = i - 1; i >= 0; i--) { 2555 char buf[PATH_MAX]; 2556 int len; 2557 2558 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2559 if (len < 0) 2560 continue; 2561 else if (len >= PATH_MAX) 2562 continue; 2563 2564 bpf_program__unpin_instance(prog, buf, i); 2565 } 2566 2567 rmdir(path); 2568 2569 return err; 2570 } 2571 2572 int bpf_program__unpin(struct bpf_program *prog, const char *path) 2573 { 2574 int i, err; 2575 2576 err = check_path(path); 2577 if (err) 2578 return err; 2579 2580 if (prog == NULL) { 2581 pr_warning("invalid program pointer\n"); 2582 return -EINVAL; 2583 } 2584 2585 if (prog->instances.nr <= 0) { 2586 pr_warning("no instances of prog %s to pin\n", 2587 prog->section_name); 2588 return -EINVAL; 2589 } 2590 2591 if (prog->instances.nr == 1) { 2592 /* don't create subdirs when pinning single instance */ 2593 return bpf_program__unpin_instance(prog, path, 0); 2594 } 2595 2596 for (i = 0; i < prog->instances.nr; i++) { 2597 char buf[PATH_MAX]; 2598 int len; 2599 2600 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2601 if (len < 0) 2602 return -EINVAL; 2603 else if (len >= PATH_MAX) 2604 return -ENAMETOOLONG; 2605 2606 err = bpf_program__unpin_instance(prog, buf, i); 2607 if (err) 2608 return err; 2609 } 2610 2611 err = rmdir(path); 2612 if (err) 2613 return -errno; 2614 2615 return 0; 2616 } 2617 2618 int bpf_map__pin(struct bpf_map *map, const char *path) 2619 { 2620 char *cp, errmsg[STRERR_BUFSIZE]; 2621 int err; 2622 2623 err = check_path(path); 2624 if (err) 2625 return err; 2626 2627 if (map == NULL) { 2628 pr_warning("invalid map pointer\n"); 2629 return -EINVAL; 2630 } 2631 2632 if (bpf_obj_pin(map->fd, path)) { 2633 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2634 pr_warning("failed to pin map: %s\n", cp); 2635 return -errno; 2636 } 2637 2638 pr_debug("pinned map '%s'\n", path); 2639 2640 return 0; 2641 } 2642 2643 int bpf_map__unpin(struct bpf_map *map, const char *path) 2644 { 2645 int err; 2646 2647 err = check_path(path); 2648 if (err) 2649 return err; 2650 2651 if (map == NULL) { 2652 pr_warning("invalid map pointer\n"); 2653 return -EINVAL; 2654 } 2655 2656 err = unlink(path); 2657 if (err != 0) 2658 return -errno; 2659 pr_debug("unpinned map '%s'\n", path); 2660 2661 return 0; 2662 } 2663 2664 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 2665 { 2666 struct bpf_map *map; 2667 int err; 2668 2669 if (!obj) 2670 return -ENOENT; 2671 2672 if (!obj->loaded) { 2673 pr_warning("object not yet loaded; load it first\n"); 2674 return -ENOENT; 2675 } 2676 2677 err = make_dir(path); 2678 if (err) 2679 return err; 2680 2681 bpf_object__for_each_map(map, obj) { 2682 char buf[PATH_MAX]; 2683 int len; 2684 2685 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2686 bpf_map__name(map)); 2687 if (len < 0) { 2688 err = -EINVAL; 2689 goto err_unpin_maps; 2690 } else if (len >= PATH_MAX) { 2691 err = -ENAMETOOLONG; 2692 goto err_unpin_maps; 2693 } 2694 2695 err = bpf_map__pin(map, buf); 2696 if (err) 2697 goto err_unpin_maps; 2698 } 2699 2700 return 0; 2701 2702 err_unpin_maps: 2703 while ((map = bpf_map__prev(map, obj))) { 2704 char buf[PATH_MAX]; 2705 int len; 2706 2707 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2708 bpf_map__name(map)); 2709 if (len < 0) 2710 continue; 2711 else if (len >= PATH_MAX) 2712 continue; 2713 2714 bpf_map__unpin(map, buf); 2715 } 2716 2717 return err; 2718 } 2719 2720 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 2721 { 2722 struct bpf_map *map; 2723 int err; 2724 2725 if (!obj) 2726 return -ENOENT; 2727 2728 bpf_object__for_each_map(map, obj) { 2729 char buf[PATH_MAX]; 2730 int len; 2731 2732 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2733 bpf_map__name(map)); 2734 if (len < 0) 2735 return -EINVAL; 2736 else if (len >= PATH_MAX) 2737 return -ENAMETOOLONG; 2738 2739 err = bpf_map__unpin(map, buf); 2740 if (err) 2741 return err; 2742 } 2743 2744 return 0; 2745 } 2746 2747 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 2748 { 2749 struct bpf_program *prog; 2750 int err; 2751 2752 if (!obj) 2753 return -ENOENT; 2754 2755 if (!obj->loaded) { 2756 pr_warning("object not yet loaded; load it first\n"); 2757 return -ENOENT; 2758 } 2759 2760 err = make_dir(path); 2761 if (err) 2762 return err; 2763 2764 bpf_object__for_each_program(prog, obj) { 2765 char buf[PATH_MAX]; 2766 int len; 2767 2768 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2769 prog->pin_name); 2770 if (len < 0) { 2771 err = -EINVAL; 2772 goto err_unpin_programs; 2773 } else if (len >= PATH_MAX) { 2774 err = -ENAMETOOLONG; 2775 goto err_unpin_programs; 2776 } 2777 2778 err = bpf_program__pin(prog, buf); 2779 if (err) 2780 goto err_unpin_programs; 2781 } 2782 2783 return 0; 2784 2785 err_unpin_programs: 2786 while ((prog = bpf_program__prev(prog, obj))) { 2787 char buf[PATH_MAX]; 2788 int len; 2789 2790 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2791 prog->pin_name); 2792 if (len < 0) 2793 continue; 2794 else if (len >= PATH_MAX) 2795 continue; 2796 2797 bpf_program__unpin(prog, buf); 2798 } 2799 2800 return err; 2801 } 2802 2803 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 2804 { 2805 struct bpf_program *prog; 2806 int err; 2807 2808 if (!obj) 2809 return -ENOENT; 2810 2811 bpf_object__for_each_program(prog, obj) { 2812 char buf[PATH_MAX]; 2813 int len; 2814 2815 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2816 prog->pin_name); 2817 if (len < 0) 2818 return -EINVAL; 2819 else if (len >= PATH_MAX) 2820 return -ENAMETOOLONG; 2821 2822 err = bpf_program__unpin(prog, buf); 2823 if (err) 2824 return err; 2825 } 2826 2827 return 0; 2828 } 2829 2830 int bpf_object__pin(struct bpf_object *obj, const char *path) 2831 { 2832 int err; 2833 2834 err = bpf_object__pin_maps(obj, path); 2835 if (err) 2836 return err; 2837 2838 err = bpf_object__pin_programs(obj, path); 2839 if (err) { 2840 bpf_object__unpin_maps(obj, path); 2841 return err; 2842 } 2843 2844 return 0; 2845 } 2846 2847 void bpf_object__close(struct bpf_object *obj) 2848 { 2849 size_t i; 2850 2851 if (!obj) 2852 return; 2853 2854 if (obj->clear_priv) 2855 obj->clear_priv(obj, obj->priv); 2856 2857 bpf_object__elf_finish(obj); 2858 bpf_object__unload(obj); 2859 btf__free(obj->btf); 2860 btf_ext__free(obj->btf_ext); 2861 2862 for (i = 0; i < obj->nr_maps; i++) { 2863 zfree(&obj->maps[i].name); 2864 if (obj->maps[i].clear_priv) 2865 obj->maps[i].clear_priv(&obj->maps[i], 2866 obj->maps[i].priv); 2867 obj->maps[i].priv = NULL; 2868 obj->maps[i].clear_priv = NULL; 2869 } 2870 2871 zfree(&obj->sections.rodata); 2872 zfree(&obj->sections.data); 2873 zfree(&obj->maps); 2874 obj->nr_maps = 0; 2875 2876 if (obj->programs && obj->nr_programs) { 2877 for (i = 0; i < obj->nr_programs; i++) 2878 bpf_program__exit(&obj->programs[i]); 2879 } 2880 zfree(&obj->programs); 2881 2882 list_del(&obj->list); 2883 free(obj); 2884 } 2885 2886 struct bpf_object * 2887 bpf_object__next(struct bpf_object *prev) 2888 { 2889 struct bpf_object *next; 2890 2891 if (!prev) 2892 next = list_first_entry(&bpf_objects_list, 2893 struct bpf_object, 2894 list); 2895 else 2896 next = list_next_entry(prev, list); 2897 2898 /* Empty list is noticed here so don't need checking on entry. */ 2899 if (&next->list == &bpf_objects_list) 2900 return NULL; 2901 2902 return next; 2903 } 2904 2905 const char *bpf_object__name(struct bpf_object *obj) 2906 { 2907 return obj ? obj->path : ERR_PTR(-EINVAL); 2908 } 2909 2910 unsigned int bpf_object__kversion(struct bpf_object *obj) 2911 { 2912 return obj ? obj->kern_version : 0; 2913 } 2914 2915 struct btf *bpf_object__btf(struct bpf_object *obj) 2916 { 2917 return obj ? obj->btf : NULL; 2918 } 2919 2920 int bpf_object__btf_fd(const struct bpf_object *obj) 2921 { 2922 return obj->btf ? btf__fd(obj->btf) : -1; 2923 } 2924 2925 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 2926 bpf_object_clear_priv_t clear_priv) 2927 { 2928 if (obj->priv && obj->clear_priv) 2929 obj->clear_priv(obj, obj->priv); 2930 2931 obj->priv = priv; 2932 obj->clear_priv = clear_priv; 2933 return 0; 2934 } 2935 2936 void *bpf_object__priv(struct bpf_object *obj) 2937 { 2938 return obj ? obj->priv : ERR_PTR(-EINVAL); 2939 } 2940 2941 static struct bpf_program * 2942 __bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward) 2943 { 2944 size_t nr_programs = obj->nr_programs; 2945 ssize_t idx; 2946 2947 if (!nr_programs) 2948 return NULL; 2949 2950 if (!p) 2951 /* Iter from the beginning */ 2952 return forward ? &obj->programs[0] : 2953 &obj->programs[nr_programs - 1]; 2954 2955 if (p->obj != obj) { 2956 pr_warning("error: program handler doesn't match object\n"); 2957 return NULL; 2958 } 2959 2960 idx = (p - obj->programs) + (forward ? 1 : -1); 2961 if (idx >= obj->nr_programs || idx < 0) 2962 return NULL; 2963 return &obj->programs[idx]; 2964 } 2965 2966 struct bpf_program * 2967 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 2968 { 2969 struct bpf_program *prog = prev; 2970 2971 do { 2972 prog = __bpf_program__iter(prog, obj, true); 2973 } while (prog && bpf_program__is_function_storage(prog, obj)); 2974 2975 return prog; 2976 } 2977 2978 struct bpf_program * 2979 bpf_program__prev(struct bpf_program *next, struct bpf_object *obj) 2980 { 2981 struct bpf_program *prog = next; 2982 2983 do { 2984 prog = __bpf_program__iter(prog, obj, false); 2985 } while (prog && bpf_program__is_function_storage(prog, obj)); 2986 2987 return prog; 2988 } 2989 2990 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 2991 bpf_program_clear_priv_t clear_priv) 2992 { 2993 if (prog->priv && prog->clear_priv) 2994 prog->clear_priv(prog, prog->priv); 2995 2996 prog->priv = priv; 2997 prog->clear_priv = clear_priv; 2998 return 0; 2999 } 3000 3001 void *bpf_program__priv(struct bpf_program *prog) 3002 { 3003 return prog ? prog->priv : ERR_PTR(-EINVAL); 3004 } 3005 3006 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 3007 { 3008 prog->prog_ifindex = ifindex; 3009 } 3010 3011 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 3012 { 3013 const char *title; 3014 3015 title = prog->section_name; 3016 if (needs_copy) { 3017 title = strdup(title); 3018 if (!title) { 3019 pr_warning("failed to strdup program title\n"); 3020 return ERR_PTR(-ENOMEM); 3021 } 3022 } 3023 3024 return title; 3025 } 3026 3027 int bpf_program__fd(struct bpf_program *prog) 3028 { 3029 return bpf_program__nth_fd(prog, 0); 3030 } 3031 3032 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 3033 bpf_program_prep_t prep) 3034 { 3035 int *instances_fds; 3036 3037 if (nr_instances <= 0 || !prep) 3038 return -EINVAL; 3039 3040 if (prog->instances.nr > 0 || prog->instances.fds) { 3041 pr_warning("Can't set pre-processor after loading\n"); 3042 return -EINVAL; 3043 } 3044 3045 instances_fds = malloc(sizeof(int) * nr_instances); 3046 if (!instances_fds) { 3047 pr_warning("alloc memory failed for fds\n"); 3048 return -ENOMEM; 3049 } 3050 3051 /* fill all fd with -1 */ 3052 memset(instances_fds, -1, sizeof(int) * nr_instances); 3053 3054 prog->instances.nr = nr_instances; 3055 prog->instances.fds = instances_fds; 3056 prog->preprocessor = prep; 3057 return 0; 3058 } 3059 3060 int bpf_program__nth_fd(struct bpf_program *prog, int n) 3061 { 3062 int fd; 3063 3064 if (!prog) 3065 return -EINVAL; 3066 3067 if (n >= prog->instances.nr || n < 0) { 3068 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 3069 n, prog->section_name, prog->instances.nr); 3070 return -EINVAL; 3071 } 3072 3073 fd = prog->instances.fds[n]; 3074 if (fd < 0) { 3075 pr_warning("%dth instance of program '%s' is invalid\n", 3076 n, prog->section_name); 3077 return -ENOENT; 3078 } 3079 3080 return fd; 3081 } 3082 3083 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 3084 { 3085 prog->type = type; 3086 } 3087 3088 static bool bpf_program__is_type(struct bpf_program *prog, 3089 enum bpf_prog_type type) 3090 { 3091 return prog ? (prog->type == type) : false; 3092 } 3093 3094 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 3095 int bpf_program__set_##NAME(struct bpf_program *prog) \ 3096 { \ 3097 if (!prog) \ 3098 return -EINVAL; \ 3099 bpf_program__set_type(prog, TYPE); \ 3100 return 0; \ 3101 } \ 3102 \ 3103 bool bpf_program__is_##NAME(struct bpf_program *prog) \ 3104 { \ 3105 return bpf_program__is_type(prog, TYPE); \ 3106 } \ 3107 3108 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 3109 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 3110 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 3111 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 3112 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 3113 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 3114 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 3115 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 3116 3117 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 3118 enum bpf_attach_type type) 3119 { 3120 prog->expected_attach_type = type; 3121 } 3122 3123 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ 3124 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } 3125 3126 /* Programs that can NOT be attached. */ 3127 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) 3128 3129 /* Programs that can be attached. */ 3130 #define BPF_APROG_SEC(string, ptype, atype) \ 3131 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) 3132 3133 /* Programs that must specify expected attach type at load time. */ 3134 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 3135 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) 3136 3137 /* Programs that can be attached but attach type can't be identified by section 3138 * name. Kept for backward compatibility. 3139 */ 3140 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 3141 3142 static const struct { 3143 const char *sec; 3144 size_t len; 3145 enum bpf_prog_type prog_type; 3146 enum bpf_attach_type expected_attach_type; 3147 int is_attachable; 3148 enum bpf_attach_type attach_type; 3149 } section_names[] = { 3150 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 3151 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 3152 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 3153 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 3154 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 3155 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 3156 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), 3157 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 3158 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 3159 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 3160 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 3161 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 3162 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 3163 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 3164 BPF_CGROUP_INET_INGRESS), 3165 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 3166 BPF_CGROUP_INET_EGRESS), 3167 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 3168 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 3169 BPF_CGROUP_INET_SOCK_CREATE), 3170 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 3171 BPF_CGROUP_INET4_POST_BIND), 3172 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 3173 BPF_CGROUP_INET6_POST_BIND), 3174 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 3175 BPF_CGROUP_DEVICE), 3176 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 3177 BPF_CGROUP_SOCK_OPS), 3178 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 3179 BPF_SK_SKB_STREAM_PARSER), 3180 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 3181 BPF_SK_SKB_STREAM_VERDICT), 3182 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 3183 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 3184 BPF_SK_MSG_VERDICT), 3185 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 3186 BPF_LIRC_MODE2), 3187 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 3188 BPF_FLOW_DISSECTOR), 3189 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3190 BPF_CGROUP_INET4_BIND), 3191 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3192 BPF_CGROUP_INET6_BIND), 3193 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3194 BPF_CGROUP_INET4_CONNECT), 3195 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3196 BPF_CGROUP_INET6_CONNECT), 3197 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3198 BPF_CGROUP_UDP4_SENDMSG), 3199 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3200 BPF_CGROUP_UDP6_SENDMSG), 3201 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3202 BPF_CGROUP_UDP4_RECVMSG), 3203 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3204 BPF_CGROUP_UDP6_RECVMSG), 3205 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 3206 BPF_CGROUP_SYSCTL), 3207 }; 3208 3209 #undef BPF_PROG_SEC_IMPL 3210 #undef BPF_PROG_SEC 3211 #undef BPF_APROG_SEC 3212 #undef BPF_EAPROG_SEC 3213 #undef BPF_APROG_COMPAT 3214 3215 #define MAX_TYPE_NAME_SIZE 32 3216 3217 static char *libbpf_get_type_names(bool attach_type) 3218 { 3219 int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; 3220 char *buf; 3221 3222 buf = malloc(len); 3223 if (!buf) 3224 return NULL; 3225 3226 buf[0] = '\0'; 3227 /* Forge string buf with all available names */ 3228 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3229 if (attach_type && !section_names[i].is_attachable) 3230 continue; 3231 3232 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { 3233 free(buf); 3234 return NULL; 3235 } 3236 strcat(buf, " "); 3237 strcat(buf, section_names[i].sec); 3238 } 3239 3240 return buf; 3241 } 3242 3243 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 3244 enum bpf_attach_type *expected_attach_type) 3245 { 3246 char *type_names; 3247 int i; 3248 3249 if (!name) 3250 return -EINVAL; 3251 3252 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3253 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3254 continue; 3255 *prog_type = section_names[i].prog_type; 3256 *expected_attach_type = section_names[i].expected_attach_type; 3257 return 0; 3258 } 3259 pr_warning("failed to guess program type based on ELF section name '%s'\n", name); 3260 type_names = libbpf_get_type_names(false); 3261 if (type_names != NULL) { 3262 pr_info("supported section(type) names are:%s\n", type_names); 3263 free(type_names); 3264 } 3265 3266 return -EINVAL; 3267 } 3268 3269 int libbpf_attach_type_by_name(const char *name, 3270 enum bpf_attach_type *attach_type) 3271 { 3272 char *type_names; 3273 int i; 3274 3275 if (!name) 3276 return -EINVAL; 3277 3278 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3279 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3280 continue; 3281 if (!section_names[i].is_attachable) 3282 return -EINVAL; 3283 *attach_type = section_names[i].attach_type; 3284 return 0; 3285 } 3286 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); 3287 type_names = libbpf_get_type_names(true); 3288 if (type_names != NULL) { 3289 pr_info("attachable section(type) names are:%s\n", type_names); 3290 free(type_names); 3291 } 3292 3293 return -EINVAL; 3294 } 3295 3296 static int 3297 bpf_program__identify_section(struct bpf_program *prog, 3298 enum bpf_prog_type *prog_type, 3299 enum bpf_attach_type *expected_attach_type) 3300 { 3301 return libbpf_prog_type_by_name(prog->section_name, prog_type, 3302 expected_attach_type); 3303 } 3304 3305 int bpf_map__fd(struct bpf_map *map) 3306 { 3307 return map ? map->fd : -EINVAL; 3308 } 3309 3310 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 3311 { 3312 return map ? &map->def : ERR_PTR(-EINVAL); 3313 } 3314 3315 const char *bpf_map__name(struct bpf_map *map) 3316 { 3317 return map ? map->name : NULL; 3318 } 3319 3320 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 3321 { 3322 return map ? map->btf_key_type_id : 0; 3323 } 3324 3325 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 3326 { 3327 return map ? map->btf_value_type_id : 0; 3328 } 3329 3330 int bpf_map__set_priv(struct bpf_map *map, void *priv, 3331 bpf_map_clear_priv_t clear_priv) 3332 { 3333 if (!map) 3334 return -EINVAL; 3335 3336 if (map->priv) { 3337 if (map->clear_priv) 3338 map->clear_priv(map, map->priv); 3339 } 3340 3341 map->priv = priv; 3342 map->clear_priv = clear_priv; 3343 return 0; 3344 } 3345 3346 void *bpf_map__priv(struct bpf_map *map) 3347 { 3348 return map ? map->priv : ERR_PTR(-EINVAL); 3349 } 3350 3351 bool bpf_map__is_offload_neutral(struct bpf_map *map) 3352 { 3353 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 3354 } 3355 3356 bool bpf_map__is_internal(struct bpf_map *map) 3357 { 3358 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 3359 } 3360 3361 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 3362 { 3363 map->map_ifindex = ifindex; 3364 } 3365 3366 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 3367 { 3368 if (!bpf_map_type__is_map_in_map(map->def.type)) { 3369 pr_warning("error: unsupported map type\n"); 3370 return -EINVAL; 3371 } 3372 if (map->inner_map_fd != -1) { 3373 pr_warning("error: inner_map_fd already specified\n"); 3374 return -EINVAL; 3375 } 3376 map->inner_map_fd = fd; 3377 return 0; 3378 } 3379 3380 static struct bpf_map * 3381 __bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i) 3382 { 3383 ssize_t idx; 3384 struct bpf_map *s, *e; 3385 3386 if (!obj || !obj->maps) 3387 return NULL; 3388 3389 s = obj->maps; 3390 e = obj->maps + obj->nr_maps; 3391 3392 if ((m < s) || (m >= e)) { 3393 pr_warning("error in %s: map handler doesn't belong to object\n", 3394 __func__); 3395 return NULL; 3396 } 3397 3398 idx = (m - obj->maps) + i; 3399 if (idx >= obj->nr_maps || idx < 0) 3400 return NULL; 3401 return &obj->maps[idx]; 3402 } 3403 3404 struct bpf_map * 3405 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 3406 { 3407 if (prev == NULL) 3408 return obj->maps; 3409 3410 return __bpf_map__iter(prev, obj, 1); 3411 } 3412 3413 struct bpf_map * 3414 bpf_map__prev(struct bpf_map *next, struct bpf_object *obj) 3415 { 3416 if (next == NULL) { 3417 if (!obj->nr_maps) 3418 return NULL; 3419 return obj->maps + obj->nr_maps - 1; 3420 } 3421 3422 return __bpf_map__iter(next, obj, -1); 3423 } 3424 3425 struct bpf_map * 3426 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 3427 { 3428 struct bpf_map *pos; 3429 3430 bpf_object__for_each_map(pos, obj) { 3431 if (pos->name && !strcmp(pos->name, name)) 3432 return pos; 3433 } 3434 return NULL; 3435 } 3436 3437 int 3438 bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name) 3439 { 3440 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 3441 } 3442 3443 struct bpf_map * 3444 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 3445 { 3446 int i; 3447 3448 for (i = 0; i < obj->nr_maps; i++) { 3449 if (obj->maps[i].offset == offset) 3450 return &obj->maps[i]; 3451 } 3452 return ERR_PTR(-ENOENT); 3453 } 3454 3455 long libbpf_get_error(const void *ptr) 3456 { 3457 return PTR_ERR_OR_ZERO(ptr); 3458 } 3459 3460 int bpf_prog_load(const char *file, enum bpf_prog_type type, 3461 struct bpf_object **pobj, int *prog_fd) 3462 { 3463 struct bpf_prog_load_attr attr; 3464 3465 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 3466 attr.file = file; 3467 attr.prog_type = type; 3468 attr.expected_attach_type = 0; 3469 3470 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 3471 } 3472 3473 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 3474 struct bpf_object **pobj, int *prog_fd) 3475 { 3476 struct bpf_object_open_attr open_attr = { 3477 .file = attr->file, 3478 .prog_type = attr->prog_type, 3479 }; 3480 struct bpf_program *prog, *first_prog = NULL; 3481 enum bpf_attach_type expected_attach_type; 3482 enum bpf_prog_type prog_type; 3483 struct bpf_object *obj; 3484 struct bpf_map *map; 3485 int err; 3486 3487 if (!attr) 3488 return -EINVAL; 3489 if (!attr->file) 3490 return -EINVAL; 3491 3492 obj = bpf_object__open_xattr(&open_attr); 3493 if (IS_ERR_OR_NULL(obj)) 3494 return -ENOENT; 3495 3496 bpf_object__for_each_program(prog, obj) { 3497 /* 3498 * If type is not specified, try to guess it based on 3499 * section name. 3500 */ 3501 prog_type = attr->prog_type; 3502 prog->prog_ifindex = attr->ifindex; 3503 expected_attach_type = attr->expected_attach_type; 3504 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 3505 err = bpf_program__identify_section(prog, &prog_type, 3506 &expected_attach_type); 3507 if (err < 0) { 3508 bpf_object__close(obj); 3509 return -EINVAL; 3510 } 3511 } 3512 3513 bpf_program__set_type(prog, prog_type); 3514 bpf_program__set_expected_attach_type(prog, 3515 expected_attach_type); 3516 3517 prog->log_level = attr->log_level; 3518 prog->prog_flags = attr->prog_flags; 3519 if (!first_prog) 3520 first_prog = prog; 3521 } 3522 3523 bpf_object__for_each_map(map, obj) { 3524 if (!bpf_map__is_offload_neutral(map)) 3525 map->map_ifindex = attr->ifindex; 3526 } 3527 3528 if (!first_prog) { 3529 pr_warning("object file doesn't contain bpf program\n"); 3530 bpf_object__close(obj); 3531 return -ENOENT; 3532 } 3533 3534 err = bpf_object__load(obj); 3535 if (err) { 3536 bpf_object__close(obj); 3537 return -EINVAL; 3538 } 3539 3540 *pobj = obj; 3541 *prog_fd = bpf_program__fd(first_prog); 3542 return 0; 3543 } 3544 3545 enum bpf_perf_event_ret 3546 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 3547 void **copy_mem, size_t *copy_size, 3548 bpf_perf_event_print_t fn, void *private_data) 3549 { 3550 struct perf_event_mmap_page *header = mmap_mem; 3551 __u64 data_head = ring_buffer_read_head(header); 3552 __u64 data_tail = header->data_tail; 3553 void *base = ((__u8 *)header) + page_size; 3554 int ret = LIBBPF_PERF_EVENT_CONT; 3555 struct perf_event_header *ehdr; 3556 size_t ehdr_size; 3557 3558 while (data_head != data_tail) { 3559 ehdr = base + (data_tail & (mmap_size - 1)); 3560 ehdr_size = ehdr->size; 3561 3562 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 3563 void *copy_start = ehdr; 3564 size_t len_first = base + mmap_size - copy_start; 3565 size_t len_secnd = ehdr_size - len_first; 3566 3567 if (*copy_size < ehdr_size) { 3568 free(*copy_mem); 3569 *copy_mem = malloc(ehdr_size); 3570 if (!*copy_mem) { 3571 *copy_size = 0; 3572 ret = LIBBPF_PERF_EVENT_ERROR; 3573 break; 3574 } 3575 *copy_size = ehdr_size; 3576 } 3577 3578 memcpy(*copy_mem, copy_start, len_first); 3579 memcpy(*copy_mem + len_first, base, len_secnd); 3580 ehdr = *copy_mem; 3581 } 3582 3583 ret = fn(ehdr, private_data); 3584 data_tail += ehdr_size; 3585 if (ret != LIBBPF_PERF_EVENT_CONT) 3586 break; 3587 } 3588 3589 ring_buffer_write_tail(header, data_tail); 3590 return ret; 3591 } 3592 3593 struct bpf_prog_info_array_desc { 3594 int array_offset; /* e.g. offset of jited_prog_insns */ 3595 int count_offset; /* e.g. offset of jited_prog_len */ 3596 int size_offset; /* > 0: offset of rec size, 3597 * < 0: fix size of -size_offset 3598 */ 3599 }; 3600 3601 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 3602 [BPF_PROG_INFO_JITED_INSNS] = { 3603 offsetof(struct bpf_prog_info, jited_prog_insns), 3604 offsetof(struct bpf_prog_info, jited_prog_len), 3605 -1, 3606 }, 3607 [BPF_PROG_INFO_XLATED_INSNS] = { 3608 offsetof(struct bpf_prog_info, xlated_prog_insns), 3609 offsetof(struct bpf_prog_info, xlated_prog_len), 3610 -1, 3611 }, 3612 [BPF_PROG_INFO_MAP_IDS] = { 3613 offsetof(struct bpf_prog_info, map_ids), 3614 offsetof(struct bpf_prog_info, nr_map_ids), 3615 -(int)sizeof(__u32), 3616 }, 3617 [BPF_PROG_INFO_JITED_KSYMS] = { 3618 offsetof(struct bpf_prog_info, jited_ksyms), 3619 offsetof(struct bpf_prog_info, nr_jited_ksyms), 3620 -(int)sizeof(__u64), 3621 }, 3622 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 3623 offsetof(struct bpf_prog_info, jited_func_lens), 3624 offsetof(struct bpf_prog_info, nr_jited_func_lens), 3625 -(int)sizeof(__u32), 3626 }, 3627 [BPF_PROG_INFO_FUNC_INFO] = { 3628 offsetof(struct bpf_prog_info, func_info), 3629 offsetof(struct bpf_prog_info, nr_func_info), 3630 offsetof(struct bpf_prog_info, func_info_rec_size), 3631 }, 3632 [BPF_PROG_INFO_LINE_INFO] = { 3633 offsetof(struct bpf_prog_info, line_info), 3634 offsetof(struct bpf_prog_info, nr_line_info), 3635 offsetof(struct bpf_prog_info, line_info_rec_size), 3636 }, 3637 [BPF_PROG_INFO_JITED_LINE_INFO] = { 3638 offsetof(struct bpf_prog_info, jited_line_info), 3639 offsetof(struct bpf_prog_info, nr_jited_line_info), 3640 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 3641 }, 3642 [BPF_PROG_INFO_PROG_TAGS] = { 3643 offsetof(struct bpf_prog_info, prog_tags), 3644 offsetof(struct bpf_prog_info, nr_prog_tags), 3645 -(int)sizeof(__u8) * BPF_TAG_SIZE, 3646 }, 3647 3648 }; 3649 3650 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) 3651 { 3652 __u32 *array = (__u32 *)info; 3653 3654 if (offset >= 0) 3655 return array[offset / sizeof(__u32)]; 3656 return -(int)offset; 3657 } 3658 3659 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) 3660 { 3661 __u64 *array = (__u64 *)info; 3662 3663 if (offset >= 0) 3664 return array[offset / sizeof(__u64)]; 3665 return -(int)offset; 3666 } 3667 3668 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 3669 __u32 val) 3670 { 3671 __u32 *array = (__u32 *)info; 3672 3673 if (offset >= 0) 3674 array[offset / sizeof(__u32)] = val; 3675 } 3676 3677 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 3678 __u64 val) 3679 { 3680 __u64 *array = (__u64 *)info; 3681 3682 if (offset >= 0) 3683 array[offset / sizeof(__u64)] = val; 3684 } 3685 3686 struct bpf_prog_info_linear * 3687 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 3688 { 3689 struct bpf_prog_info_linear *info_linear; 3690 struct bpf_prog_info info = {}; 3691 __u32 info_len = sizeof(info); 3692 __u32 data_len = 0; 3693 int i, err; 3694 void *ptr; 3695 3696 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 3697 return ERR_PTR(-EINVAL); 3698 3699 /* step 1: get array dimensions */ 3700 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 3701 if (err) { 3702 pr_debug("can't get prog info: %s", strerror(errno)); 3703 return ERR_PTR(-EFAULT); 3704 } 3705 3706 /* step 2: calculate total size of all arrays */ 3707 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3708 bool include_array = (arrays & (1UL << i)) > 0; 3709 struct bpf_prog_info_array_desc *desc; 3710 __u32 count, size; 3711 3712 desc = bpf_prog_info_array_desc + i; 3713 3714 /* kernel is too old to support this field */ 3715 if (info_len < desc->array_offset + sizeof(__u32) || 3716 info_len < desc->count_offset + sizeof(__u32) || 3717 (desc->size_offset > 0 && info_len < desc->size_offset)) 3718 include_array = false; 3719 3720 if (!include_array) { 3721 arrays &= ~(1UL << i); /* clear the bit */ 3722 continue; 3723 } 3724 3725 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3726 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3727 3728 data_len += count * size; 3729 } 3730 3731 /* step 3: allocate continuous memory */ 3732 data_len = roundup(data_len, sizeof(__u64)); 3733 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 3734 if (!info_linear) 3735 return ERR_PTR(-ENOMEM); 3736 3737 /* step 4: fill data to info_linear->info */ 3738 info_linear->arrays = arrays; 3739 memset(&info_linear->info, 0, sizeof(info)); 3740 ptr = info_linear->data; 3741 3742 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3743 struct bpf_prog_info_array_desc *desc; 3744 __u32 count, size; 3745 3746 if ((arrays & (1UL << i)) == 0) 3747 continue; 3748 3749 desc = bpf_prog_info_array_desc + i; 3750 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3751 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3752 bpf_prog_info_set_offset_u32(&info_linear->info, 3753 desc->count_offset, count); 3754 bpf_prog_info_set_offset_u32(&info_linear->info, 3755 desc->size_offset, size); 3756 bpf_prog_info_set_offset_u64(&info_linear->info, 3757 desc->array_offset, 3758 ptr_to_u64(ptr)); 3759 ptr += count * size; 3760 } 3761 3762 /* step 5: call syscall again to get required arrays */ 3763 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 3764 if (err) { 3765 pr_debug("can't get prog info: %s", strerror(errno)); 3766 free(info_linear); 3767 return ERR_PTR(-EFAULT); 3768 } 3769 3770 /* step 6: verify the data */ 3771 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3772 struct bpf_prog_info_array_desc *desc; 3773 __u32 v1, v2; 3774 3775 if ((arrays & (1UL << i)) == 0) 3776 continue; 3777 3778 desc = bpf_prog_info_array_desc + i; 3779 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3780 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3781 desc->count_offset); 3782 if (v1 != v2) 3783 pr_warning("%s: mismatch in element count\n", __func__); 3784 3785 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3786 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3787 desc->size_offset); 3788 if (v1 != v2) 3789 pr_warning("%s: mismatch in rec size\n", __func__); 3790 } 3791 3792 /* step 7: update info_len and data_len */ 3793 info_linear->info_len = sizeof(struct bpf_prog_info); 3794 info_linear->data_len = data_len; 3795 3796 return info_linear; 3797 } 3798 3799 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 3800 { 3801 int i; 3802 3803 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3804 struct bpf_prog_info_array_desc *desc; 3805 __u64 addr, offs; 3806 3807 if ((info_linear->arrays & (1UL << i)) == 0) 3808 continue; 3809 3810 desc = bpf_prog_info_array_desc + i; 3811 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 3812 desc->array_offset); 3813 offs = addr - ptr_to_u64(info_linear->data); 3814 bpf_prog_info_set_offset_u64(&info_linear->info, 3815 desc->array_offset, offs); 3816 } 3817 } 3818 3819 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 3820 { 3821 int i; 3822 3823 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3824 struct bpf_prog_info_array_desc *desc; 3825 __u64 addr, offs; 3826 3827 if ((info_linear->arrays & (1UL << i)) == 0) 3828 continue; 3829 3830 desc = bpf_prog_info_array_desc + i; 3831 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 3832 desc->array_offset); 3833 addr = offs + ptr_to_u64(info_linear->data); 3834 bpf_prog_info_set_offset_u64(&info_linear->info, 3835 desc->array_offset, addr); 3836 } 3837 } 3838