1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <string.h> 22 #include <unistd.h> 23 #include <fcntl.h> 24 #include <errno.h> 25 #include <asm/unistd.h> 26 #include <linux/err.h> 27 #include <linux/kernel.h> 28 #include <linux/bpf.h> 29 #include <linux/btf.h> 30 #include <linux/filter.h> 31 #include <linux/list.h> 32 #include <linux/limits.h> 33 #include <linux/perf_event.h> 34 #include <linux/ring_buffer.h> 35 #include <sys/stat.h> 36 #include <sys/types.h> 37 #include <sys/vfs.h> 38 #include <tools/libc_compat.h> 39 #include <libelf.h> 40 #include <gelf.h> 41 42 #include "libbpf.h" 43 #include "bpf.h" 44 #include "btf.h" 45 #include "str_error.h" 46 #include "libbpf_internal.h" 47 48 #ifndef EM_BPF 49 #define EM_BPF 247 50 #endif 51 52 #ifndef BPF_FS_MAGIC 53 #define BPF_FS_MAGIC 0xcafe4a11 54 #endif 55 56 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 57 * compilation if user enables corresponding warning. Disable it explicitly. 58 */ 59 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 60 61 #define __printf(a, b) __attribute__((format(printf, a, b))) 62 63 static int __base_pr(enum libbpf_print_level level, const char *format, 64 va_list args) 65 { 66 if (level == LIBBPF_DEBUG) 67 return 0; 68 69 return vfprintf(stderr, format, args); 70 } 71 72 static libbpf_print_fn_t __libbpf_pr = __base_pr; 73 74 void libbpf_set_print(libbpf_print_fn_t fn) 75 { 76 __libbpf_pr = fn; 77 } 78 79 __printf(2, 3) 80 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 81 { 82 va_list args; 83 84 if (!__libbpf_pr) 85 return; 86 87 va_start(args, format); 88 __libbpf_pr(level, format, args); 89 va_end(args); 90 } 91 92 #define STRERR_BUFSIZE 128 93 94 #define CHECK_ERR(action, err, out) do { \ 95 err = action; \ 96 if (err) \ 97 goto out; \ 98 } while(0) 99 100 101 /* Copied from tools/perf/util/util.h */ 102 #ifndef zfree 103 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 104 #endif 105 106 #ifndef zclose 107 # define zclose(fd) ({ \ 108 int ___err = 0; \ 109 if ((fd) >= 0) \ 110 ___err = close((fd)); \ 111 fd = -1; \ 112 ___err; }) 113 #endif 114 115 #ifdef HAVE_LIBELF_MMAP_SUPPORT 116 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 117 #else 118 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 119 #endif 120 121 static inline __u64 ptr_to_u64(const void *ptr) 122 { 123 return (__u64) (unsigned long) ptr; 124 } 125 126 struct bpf_capabilities { 127 /* v4.14: kernel support for program & map names. */ 128 __u32 name:1; 129 /* v5.2: kernel support for global data sections. */ 130 __u32 global_data:1; 131 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 132 __u32 btf_func:1; 133 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 134 __u32 btf_datasec:1; 135 }; 136 137 /* 138 * bpf_prog should be a better name but it has been used in 139 * linux/filter.h. 140 */ 141 struct bpf_program { 142 /* Index in elf obj file, for relocation use. */ 143 int idx; 144 char *name; 145 int prog_ifindex; 146 char *section_name; 147 /* section_name with / replaced by _; makes recursive pinning 148 * in bpf_object__pin_programs easier 149 */ 150 char *pin_name; 151 struct bpf_insn *insns; 152 size_t insns_cnt, main_prog_cnt; 153 enum bpf_prog_type type; 154 155 struct reloc_desc { 156 enum { 157 RELO_LD64, 158 RELO_CALL, 159 RELO_DATA, 160 } type; 161 int insn_idx; 162 union { 163 int map_idx; 164 int text_off; 165 }; 166 } *reloc_desc; 167 int nr_reloc; 168 int log_level; 169 170 struct { 171 int nr; 172 int *fds; 173 } instances; 174 bpf_program_prep_t preprocessor; 175 176 struct bpf_object *obj; 177 void *priv; 178 bpf_program_clear_priv_t clear_priv; 179 180 enum bpf_attach_type expected_attach_type; 181 int btf_fd; 182 void *func_info; 183 __u32 func_info_rec_size; 184 __u32 func_info_cnt; 185 186 struct bpf_capabilities *caps; 187 188 void *line_info; 189 __u32 line_info_rec_size; 190 __u32 line_info_cnt; 191 __u32 prog_flags; 192 }; 193 194 enum libbpf_map_type { 195 LIBBPF_MAP_UNSPEC, 196 LIBBPF_MAP_DATA, 197 LIBBPF_MAP_BSS, 198 LIBBPF_MAP_RODATA, 199 }; 200 201 static const char * const libbpf_type_to_btf_name[] = { 202 [LIBBPF_MAP_DATA] = ".data", 203 [LIBBPF_MAP_BSS] = ".bss", 204 [LIBBPF_MAP_RODATA] = ".rodata", 205 }; 206 207 struct bpf_map { 208 int fd; 209 char *name; 210 size_t offset; 211 int map_ifindex; 212 int inner_map_fd; 213 struct bpf_map_def def; 214 __u32 btf_key_type_id; 215 __u32 btf_value_type_id; 216 void *priv; 217 bpf_map_clear_priv_t clear_priv; 218 enum libbpf_map_type libbpf_type; 219 }; 220 221 struct bpf_secdata { 222 void *rodata; 223 void *data; 224 }; 225 226 static LIST_HEAD(bpf_objects_list); 227 228 struct bpf_object { 229 char name[BPF_OBJ_NAME_LEN]; 230 char license[64]; 231 __u32 kern_version; 232 233 struct bpf_program *programs; 234 size_t nr_programs; 235 struct bpf_map *maps; 236 size_t nr_maps; 237 struct bpf_secdata sections; 238 239 bool loaded; 240 bool has_pseudo_calls; 241 242 /* 243 * Information when doing elf related work. Only valid if fd 244 * is valid. 245 */ 246 struct { 247 int fd; 248 void *obj_buf; 249 size_t obj_buf_sz; 250 Elf *elf; 251 GElf_Ehdr ehdr; 252 Elf_Data *symbols; 253 Elf_Data *data; 254 Elf_Data *rodata; 255 Elf_Data *bss; 256 size_t strtabidx; 257 struct { 258 GElf_Shdr shdr; 259 Elf_Data *data; 260 } *reloc; 261 int nr_reloc; 262 int maps_shndx; 263 int text_shndx; 264 int data_shndx; 265 int rodata_shndx; 266 int bss_shndx; 267 } efile; 268 /* 269 * All loaded bpf_object is linked in a list, which is 270 * hidden to caller. bpf_objects__<func> handlers deal with 271 * all objects. 272 */ 273 struct list_head list; 274 275 struct btf *btf; 276 struct btf_ext *btf_ext; 277 278 void *priv; 279 bpf_object_clear_priv_t clear_priv; 280 281 struct bpf_capabilities caps; 282 283 char path[]; 284 }; 285 #define obj_elf_valid(o) ((o)->efile.elf) 286 287 void bpf_program__unload(struct bpf_program *prog) 288 { 289 int i; 290 291 if (!prog) 292 return; 293 294 /* 295 * If the object is opened but the program was never loaded, 296 * it is possible that prog->instances.nr == -1. 297 */ 298 if (prog->instances.nr > 0) { 299 for (i = 0; i < prog->instances.nr; i++) 300 zclose(prog->instances.fds[i]); 301 } else if (prog->instances.nr != -1) { 302 pr_warning("Internal error: instances.nr is %d\n", 303 prog->instances.nr); 304 } 305 306 prog->instances.nr = -1; 307 zfree(&prog->instances.fds); 308 309 zclose(prog->btf_fd); 310 zfree(&prog->func_info); 311 zfree(&prog->line_info); 312 } 313 314 static void bpf_program__exit(struct bpf_program *prog) 315 { 316 if (!prog) 317 return; 318 319 if (prog->clear_priv) 320 prog->clear_priv(prog, prog->priv); 321 322 prog->priv = NULL; 323 prog->clear_priv = NULL; 324 325 bpf_program__unload(prog); 326 zfree(&prog->name); 327 zfree(&prog->section_name); 328 zfree(&prog->pin_name); 329 zfree(&prog->insns); 330 zfree(&prog->reloc_desc); 331 332 prog->nr_reloc = 0; 333 prog->insns_cnt = 0; 334 prog->idx = -1; 335 } 336 337 static char *__bpf_program__pin_name(struct bpf_program *prog) 338 { 339 char *name, *p; 340 341 name = p = strdup(prog->section_name); 342 while ((p = strchr(p, '/'))) 343 *p = '_'; 344 345 return name; 346 } 347 348 static int 349 bpf_program__init(void *data, size_t size, char *section_name, int idx, 350 struct bpf_program *prog) 351 { 352 const size_t bpf_insn_sz = sizeof(struct bpf_insn); 353 354 if (size == 0 || size % bpf_insn_sz) { 355 pr_warning("corrupted section '%s', size: %zu\n", 356 section_name, size); 357 return -EINVAL; 358 } 359 360 memset(prog, 0, sizeof(*prog)); 361 362 prog->section_name = strdup(section_name); 363 if (!prog->section_name) { 364 pr_warning("failed to alloc name for prog under section(%d) %s\n", 365 idx, section_name); 366 goto errout; 367 } 368 369 prog->pin_name = __bpf_program__pin_name(prog); 370 if (!prog->pin_name) { 371 pr_warning("failed to alloc pin name for prog under section(%d) %s\n", 372 idx, section_name); 373 goto errout; 374 } 375 376 prog->insns = malloc(size); 377 if (!prog->insns) { 378 pr_warning("failed to alloc insns for prog under section %s\n", 379 section_name); 380 goto errout; 381 } 382 prog->insns_cnt = size / bpf_insn_sz; 383 memcpy(prog->insns, data, size); 384 prog->idx = idx; 385 prog->instances.fds = NULL; 386 prog->instances.nr = -1; 387 prog->type = BPF_PROG_TYPE_UNSPEC; 388 prog->btf_fd = -1; 389 390 return 0; 391 errout: 392 bpf_program__exit(prog); 393 return -ENOMEM; 394 } 395 396 static int 397 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 398 char *section_name, int idx) 399 { 400 struct bpf_program prog, *progs; 401 int nr_progs, err; 402 403 err = bpf_program__init(data, size, section_name, idx, &prog); 404 if (err) 405 return err; 406 407 prog.caps = &obj->caps; 408 progs = obj->programs; 409 nr_progs = obj->nr_programs; 410 411 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 412 if (!progs) { 413 /* 414 * In this case the original obj->programs 415 * is still valid, so don't need special treat for 416 * bpf_close_object(). 417 */ 418 pr_warning("failed to alloc a new program under section '%s'\n", 419 section_name); 420 bpf_program__exit(&prog); 421 return -ENOMEM; 422 } 423 424 pr_debug("found program %s\n", prog.section_name); 425 obj->programs = progs; 426 obj->nr_programs = nr_progs + 1; 427 prog.obj = obj; 428 progs[nr_progs] = prog; 429 return 0; 430 } 431 432 static int 433 bpf_object__init_prog_names(struct bpf_object *obj) 434 { 435 Elf_Data *symbols = obj->efile.symbols; 436 struct bpf_program *prog; 437 size_t pi, si; 438 439 for (pi = 0; pi < obj->nr_programs; pi++) { 440 const char *name = NULL; 441 442 prog = &obj->programs[pi]; 443 444 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 445 si++) { 446 GElf_Sym sym; 447 448 if (!gelf_getsym(symbols, si, &sym)) 449 continue; 450 if (sym.st_shndx != prog->idx) 451 continue; 452 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 453 continue; 454 455 name = elf_strptr(obj->efile.elf, 456 obj->efile.strtabidx, 457 sym.st_name); 458 if (!name) { 459 pr_warning("failed to get sym name string for prog %s\n", 460 prog->section_name); 461 return -LIBBPF_ERRNO__LIBELF; 462 } 463 } 464 465 if (!name && prog->idx == obj->efile.text_shndx) 466 name = ".text"; 467 468 if (!name) { 469 pr_warning("failed to find sym for prog %s\n", 470 prog->section_name); 471 return -EINVAL; 472 } 473 474 prog->name = strdup(name); 475 if (!prog->name) { 476 pr_warning("failed to allocate memory for prog sym %s\n", 477 name); 478 return -ENOMEM; 479 } 480 } 481 482 return 0; 483 } 484 485 static struct bpf_object *bpf_object__new(const char *path, 486 void *obj_buf, 487 size_t obj_buf_sz) 488 { 489 struct bpf_object *obj; 490 char *end; 491 492 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 493 if (!obj) { 494 pr_warning("alloc memory failed for %s\n", path); 495 return ERR_PTR(-ENOMEM); 496 } 497 498 strcpy(obj->path, path); 499 /* Using basename() GNU version which doesn't modify arg. */ 500 strncpy(obj->name, basename((void *)path), sizeof(obj->name) - 1); 501 end = strchr(obj->name, '.'); 502 if (end) 503 *end = 0; 504 505 obj->efile.fd = -1; 506 /* 507 * Caller of this function should also call 508 * bpf_object__elf_finish() after data collection to return 509 * obj_buf to user. If not, we should duplicate the buffer to 510 * avoid user freeing them before elf finish. 511 */ 512 obj->efile.obj_buf = obj_buf; 513 obj->efile.obj_buf_sz = obj_buf_sz; 514 obj->efile.maps_shndx = -1; 515 obj->efile.data_shndx = -1; 516 obj->efile.rodata_shndx = -1; 517 obj->efile.bss_shndx = -1; 518 519 obj->loaded = false; 520 521 INIT_LIST_HEAD(&obj->list); 522 list_add(&obj->list, &bpf_objects_list); 523 return obj; 524 } 525 526 static void bpf_object__elf_finish(struct bpf_object *obj) 527 { 528 if (!obj_elf_valid(obj)) 529 return; 530 531 if (obj->efile.elf) { 532 elf_end(obj->efile.elf); 533 obj->efile.elf = NULL; 534 } 535 obj->efile.symbols = NULL; 536 obj->efile.data = NULL; 537 obj->efile.rodata = NULL; 538 obj->efile.bss = NULL; 539 540 zfree(&obj->efile.reloc); 541 obj->efile.nr_reloc = 0; 542 zclose(obj->efile.fd); 543 obj->efile.obj_buf = NULL; 544 obj->efile.obj_buf_sz = 0; 545 } 546 547 static int bpf_object__elf_init(struct bpf_object *obj) 548 { 549 int err = 0; 550 GElf_Ehdr *ep; 551 552 if (obj_elf_valid(obj)) { 553 pr_warning("elf init: internal error\n"); 554 return -LIBBPF_ERRNO__LIBELF; 555 } 556 557 if (obj->efile.obj_buf_sz > 0) { 558 /* 559 * obj_buf should have been validated by 560 * bpf_object__open_buffer(). 561 */ 562 obj->efile.elf = elf_memory(obj->efile.obj_buf, 563 obj->efile.obj_buf_sz); 564 } else { 565 obj->efile.fd = open(obj->path, O_RDONLY); 566 if (obj->efile.fd < 0) { 567 char errmsg[STRERR_BUFSIZE], *cp; 568 569 err = -errno; 570 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 571 pr_warning("failed to open %s: %s\n", obj->path, cp); 572 return err; 573 } 574 575 obj->efile.elf = elf_begin(obj->efile.fd, 576 LIBBPF_ELF_C_READ_MMAP, NULL); 577 } 578 579 if (!obj->efile.elf) { 580 pr_warning("failed to open %s as ELF file\n", obj->path); 581 err = -LIBBPF_ERRNO__LIBELF; 582 goto errout; 583 } 584 585 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 586 pr_warning("failed to get EHDR from %s\n", obj->path); 587 err = -LIBBPF_ERRNO__FORMAT; 588 goto errout; 589 } 590 ep = &obj->efile.ehdr; 591 592 /* Old LLVM set e_machine to EM_NONE */ 593 if (ep->e_type != ET_REL || 594 (ep->e_machine && ep->e_machine != EM_BPF)) { 595 pr_warning("%s is not an eBPF object file\n", obj->path); 596 err = -LIBBPF_ERRNO__FORMAT; 597 goto errout; 598 } 599 600 return 0; 601 errout: 602 bpf_object__elf_finish(obj); 603 return err; 604 } 605 606 static int bpf_object__check_endianness(struct bpf_object *obj) 607 { 608 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 609 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) 610 return 0; 611 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 612 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) 613 return 0; 614 #else 615 # error "Unrecognized __BYTE_ORDER__" 616 #endif 617 pr_warning("endianness mismatch.\n"); 618 return -LIBBPF_ERRNO__ENDIAN; 619 } 620 621 static int 622 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 623 { 624 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); 625 pr_debug("license of %s is %s\n", obj->path, obj->license); 626 return 0; 627 } 628 629 static int 630 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 631 { 632 __u32 kver; 633 634 if (size != sizeof(kver)) { 635 pr_warning("invalid kver section in %s\n", obj->path); 636 return -LIBBPF_ERRNO__FORMAT; 637 } 638 memcpy(&kver, data, sizeof(kver)); 639 obj->kern_version = kver; 640 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 641 return 0; 642 } 643 644 static int compare_bpf_map(const void *_a, const void *_b) 645 { 646 const struct bpf_map *a = _a; 647 const struct bpf_map *b = _b; 648 649 return a->offset - b->offset; 650 } 651 652 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 653 { 654 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 655 type == BPF_MAP_TYPE_HASH_OF_MAPS) 656 return true; 657 return false; 658 } 659 660 static int bpf_object_search_section_size(const struct bpf_object *obj, 661 const char *name, size_t *d_size) 662 { 663 const GElf_Ehdr *ep = &obj->efile.ehdr; 664 Elf *elf = obj->efile.elf; 665 Elf_Scn *scn = NULL; 666 int idx = 0; 667 668 while ((scn = elf_nextscn(elf, scn)) != NULL) { 669 const char *sec_name; 670 Elf_Data *data; 671 GElf_Shdr sh; 672 673 idx++; 674 if (gelf_getshdr(scn, &sh) != &sh) { 675 pr_warning("failed to get section(%d) header from %s\n", 676 idx, obj->path); 677 return -EIO; 678 } 679 680 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 681 if (!sec_name) { 682 pr_warning("failed to get section(%d) name from %s\n", 683 idx, obj->path); 684 return -EIO; 685 } 686 687 if (strcmp(name, sec_name)) 688 continue; 689 690 data = elf_getdata(scn, 0); 691 if (!data) { 692 pr_warning("failed to get section(%d) data from %s(%s)\n", 693 idx, name, obj->path); 694 return -EIO; 695 } 696 697 *d_size = data->d_size; 698 return 0; 699 } 700 701 return -ENOENT; 702 } 703 704 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 705 __u32 *size) 706 { 707 int ret = -ENOENT; 708 size_t d_size; 709 710 *size = 0; 711 if (!name) { 712 return -EINVAL; 713 } else if (!strcmp(name, ".data")) { 714 if (obj->efile.data) 715 *size = obj->efile.data->d_size; 716 } else if (!strcmp(name, ".bss")) { 717 if (obj->efile.bss) 718 *size = obj->efile.bss->d_size; 719 } else if (!strcmp(name, ".rodata")) { 720 if (obj->efile.rodata) 721 *size = obj->efile.rodata->d_size; 722 } else { 723 ret = bpf_object_search_section_size(obj, name, &d_size); 724 if (!ret) 725 *size = d_size; 726 } 727 728 return *size ? 0 : ret; 729 } 730 731 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 732 __u32 *off) 733 { 734 Elf_Data *symbols = obj->efile.symbols; 735 const char *sname; 736 size_t si; 737 738 if (!name || !off) 739 return -EINVAL; 740 741 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 742 GElf_Sym sym; 743 744 if (!gelf_getsym(symbols, si, &sym)) 745 continue; 746 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 747 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 748 continue; 749 750 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 751 sym.st_name); 752 if (!sname) { 753 pr_warning("failed to get sym name string for var %s\n", 754 name); 755 return -EIO; 756 } 757 if (strcmp(name, sname) == 0) { 758 *off = sym.st_value; 759 return 0; 760 } 761 } 762 763 return -ENOENT; 764 } 765 766 static bool bpf_object__has_maps(const struct bpf_object *obj) 767 { 768 return obj->efile.maps_shndx >= 0 || 769 obj->efile.data_shndx >= 0 || 770 obj->efile.rodata_shndx >= 0 || 771 obj->efile.bss_shndx >= 0; 772 } 773 774 static int 775 bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map, 776 enum libbpf_map_type type, Elf_Data *data, 777 void **data_buff) 778 { 779 struct bpf_map_def *def = &map->def; 780 char map_name[BPF_OBJ_NAME_LEN]; 781 782 map->libbpf_type = type; 783 map->offset = ~(typeof(map->offset))0; 784 snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, 785 libbpf_type_to_btf_name[type]); 786 map->name = strdup(map_name); 787 if (!map->name) { 788 pr_warning("failed to alloc map name\n"); 789 return -ENOMEM; 790 } 791 792 def->type = BPF_MAP_TYPE_ARRAY; 793 def->key_size = sizeof(int); 794 def->value_size = data->d_size; 795 def->max_entries = 1; 796 def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0; 797 if (data_buff) { 798 *data_buff = malloc(data->d_size); 799 if (!*data_buff) { 800 zfree(&map->name); 801 pr_warning("failed to alloc map content buffer\n"); 802 return -ENOMEM; 803 } 804 memcpy(*data_buff, data->d_buf, data->d_size); 805 } 806 807 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 808 return 0; 809 } 810 811 static int bpf_object__init_maps(struct bpf_object *obj, int flags) 812 { 813 int i, map_idx, map_def_sz = 0, nr_syms, nr_maps = 0, nr_maps_glob = 0; 814 bool strict = !(flags & MAPS_RELAX_COMPAT); 815 Elf_Data *symbols = obj->efile.symbols; 816 Elf_Data *data = NULL; 817 int ret = 0; 818 819 if (!symbols) 820 return -EINVAL; 821 nr_syms = symbols->d_size / sizeof(GElf_Sym); 822 823 if (obj->efile.maps_shndx >= 0) { 824 Elf_Scn *scn = elf_getscn(obj->efile.elf, 825 obj->efile.maps_shndx); 826 827 if (scn) 828 data = elf_getdata(scn, NULL); 829 if (!scn || !data) { 830 pr_warning("failed to get Elf_Data from map section %d\n", 831 obj->efile.maps_shndx); 832 return -EINVAL; 833 } 834 } 835 836 /* 837 * Count number of maps. Each map has a name. 838 * Array of maps is not supported: only the first element is 839 * considered. 840 * 841 * TODO: Detect array of map and report error. 842 */ 843 if (obj->caps.global_data) { 844 if (obj->efile.data_shndx >= 0) 845 nr_maps_glob++; 846 if (obj->efile.rodata_shndx >= 0) 847 nr_maps_glob++; 848 if (obj->efile.bss_shndx >= 0) 849 nr_maps_glob++; 850 } 851 852 for (i = 0; data && i < nr_syms; i++) { 853 GElf_Sym sym; 854 855 if (!gelf_getsym(symbols, i, &sym)) 856 continue; 857 if (sym.st_shndx != obj->efile.maps_shndx) 858 continue; 859 nr_maps++; 860 } 861 862 if (!nr_maps && !nr_maps_glob) 863 return 0; 864 865 /* Assume equally sized map definitions */ 866 if (data) { 867 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 868 nr_maps, data->d_size); 869 870 map_def_sz = data->d_size / nr_maps; 871 if (!data->d_size || (data->d_size % nr_maps) != 0) { 872 pr_warning("unable to determine map definition size " 873 "section %s, %d maps in %zd bytes\n", 874 obj->path, nr_maps, data->d_size); 875 return -EINVAL; 876 } 877 } 878 879 nr_maps += nr_maps_glob; 880 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 881 if (!obj->maps) { 882 pr_warning("alloc maps for object failed\n"); 883 return -ENOMEM; 884 } 885 obj->nr_maps = nr_maps; 886 887 for (i = 0; i < nr_maps; i++) { 888 /* 889 * fill all fd with -1 so won't close incorrect 890 * fd (fd=0 is stdin) when failure (zclose won't close 891 * negative fd)). 892 */ 893 obj->maps[i].fd = -1; 894 obj->maps[i].inner_map_fd = -1; 895 } 896 897 /* 898 * Fill obj->maps using data in "maps" section. 899 */ 900 for (i = 0, map_idx = 0; data && i < nr_syms; i++) { 901 GElf_Sym sym; 902 const char *map_name; 903 struct bpf_map_def *def; 904 905 if (!gelf_getsym(symbols, i, &sym)) 906 continue; 907 if (sym.st_shndx != obj->efile.maps_shndx) 908 continue; 909 910 map_name = elf_strptr(obj->efile.elf, 911 obj->efile.strtabidx, 912 sym.st_name); 913 if (!map_name) { 914 pr_warning("failed to get map #%d name sym string for obj %s\n", 915 map_idx, obj->path); 916 return -LIBBPF_ERRNO__FORMAT; 917 } 918 919 obj->maps[map_idx].libbpf_type = LIBBPF_MAP_UNSPEC; 920 obj->maps[map_idx].offset = sym.st_value; 921 if (sym.st_value + map_def_sz > data->d_size) { 922 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 923 obj->path, map_name); 924 return -EINVAL; 925 } 926 927 obj->maps[map_idx].name = strdup(map_name); 928 if (!obj->maps[map_idx].name) { 929 pr_warning("failed to alloc map name\n"); 930 return -ENOMEM; 931 } 932 pr_debug("map %d is \"%s\"\n", map_idx, 933 obj->maps[map_idx].name); 934 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 935 /* 936 * If the definition of the map in the object file fits in 937 * bpf_map_def, copy it. Any extra fields in our version 938 * of bpf_map_def will default to zero as a result of the 939 * calloc above. 940 */ 941 if (map_def_sz <= sizeof(struct bpf_map_def)) { 942 memcpy(&obj->maps[map_idx].def, def, map_def_sz); 943 } else { 944 /* 945 * Here the map structure being read is bigger than what 946 * we expect, truncate if the excess bits are all zero. 947 * If they are not zero, reject this map as 948 * incompatible. 949 */ 950 char *b; 951 for (b = ((char *)def) + sizeof(struct bpf_map_def); 952 b < ((char *)def) + map_def_sz; b++) { 953 if (*b != 0) { 954 pr_warning("maps section in %s: \"%s\" " 955 "has unrecognized, non-zero " 956 "options\n", 957 obj->path, map_name); 958 if (strict) 959 return -EINVAL; 960 } 961 } 962 memcpy(&obj->maps[map_idx].def, def, 963 sizeof(struct bpf_map_def)); 964 } 965 map_idx++; 966 } 967 968 if (!obj->caps.global_data) 969 goto finalize; 970 971 /* 972 * Populate rest of obj->maps with libbpf internal maps. 973 */ 974 if (obj->efile.data_shndx >= 0) 975 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 976 LIBBPF_MAP_DATA, 977 obj->efile.data, 978 &obj->sections.data); 979 if (!ret && obj->efile.rodata_shndx >= 0) 980 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 981 LIBBPF_MAP_RODATA, 982 obj->efile.rodata, 983 &obj->sections.rodata); 984 if (!ret && obj->efile.bss_shndx >= 0) 985 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 986 LIBBPF_MAP_BSS, 987 obj->efile.bss, NULL); 988 finalize: 989 if (!ret) 990 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), 991 compare_bpf_map); 992 return ret; 993 } 994 995 static bool section_have_execinstr(struct bpf_object *obj, int idx) 996 { 997 Elf_Scn *scn; 998 GElf_Shdr sh; 999 1000 scn = elf_getscn(obj->efile.elf, idx); 1001 if (!scn) 1002 return false; 1003 1004 if (gelf_getshdr(scn, &sh) != &sh) 1005 return false; 1006 1007 if (sh.sh_flags & SHF_EXECINSTR) 1008 return true; 1009 1010 return false; 1011 } 1012 1013 static void bpf_object__sanitize_btf(struct bpf_object *obj) 1014 { 1015 bool has_datasec = obj->caps.btf_datasec; 1016 bool has_func = obj->caps.btf_func; 1017 struct btf *btf = obj->btf; 1018 struct btf_type *t; 1019 int i, j, vlen; 1020 __u16 kind; 1021 1022 if (!obj->btf || (has_func && has_datasec)) 1023 return; 1024 1025 for (i = 1; i <= btf__get_nr_types(btf); i++) { 1026 t = (struct btf_type *)btf__type_by_id(btf, i); 1027 kind = BTF_INFO_KIND(t->info); 1028 1029 if (!has_datasec && kind == BTF_KIND_VAR) { 1030 /* replace VAR with INT */ 1031 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 1032 t->size = sizeof(int); 1033 *(int *)(t+1) = BTF_INT_ENC(0, 0, 32); 1034 } else if (!has_datasec && kind == BTF_KIND_DATASEC) { 1035 /* replace DATASEC with STRUCT */ 1036 struct btf_var_secinfo *v = (void *)(t + 1); 1037 struct btf_member *m = (void *)(t + 1); 1038 struct btf_type *vt; 1039 char *name; 1040 1041 name = (char *)btf__name_by_offset(btf, t->name_off); 1042 while (*name) { 1043 if (*name == '.') 1044 *name = '_'; 1045 name++; 1046 } 1047 1048 vlen = BTF_INFO_VLEN(t->info); 1049 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 1050 for (j = 0; j < vlen; j++, v++, m++) { 1051 /* order of field assignments is important */ 1052 m->offset = v->offset * 8; 1053 m->type = v->type; 1054 /* preserve variable name as member name */ 1055 vt = (void *)btf__type_by_id(btf, v->type); 1056 m->name_off = vt->name_off; 1057 } 1058 } else if (!has_func && kind == BTF_KIND_FUNC_PROTO) { 1059 /* replace FUNC_PROTO with ENUM */ 1060 vlen = BTF_INFO_VLEN(t->info); 1061 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 1062 t->size = sizeof(__u32); /* kernel enforced */ 1063 } else if (!has_func && kind == BTF_KIND_FUNC) { 1064 /* replace FUNC with TYPEDEF */ 1065 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 1066 } 1067 } 1068 } 1069 1070 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) 1071 { 1072 if (!obj->btf_ext) 1073 return; 1074 1075 if (!obj->caps.btf_func) { 1076 btf_ext__free(obj->btf_ext); 1077 obj->btf_ext = NULL; 1078 } 1079 } 1080 1081 static int bpf_object__elf_collect(struct bpf_object *obj, int flags) 1082 { 1083 Elf *elf = obj->efile.elf; 1084 GElf_Ehdr *ep = &obj->efile.ehdr; 1085 Elf_Data *btf_ext_data = NULL; 1086 Elf_Data *btf_data = NULL; 1087 Elf_Scn *scn = NULL; 1088 int idx = 0, err = 0; 1089 1090 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1091 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 1092 pr_warning("failed to get e_shstrndx from %s\n", obj->path); 1093 return -LIBBPF_ERRNO__FORMAT; 1094 } 1095 1096 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1097 char *name; 1098 GElf_Shdr sh; 1099 Elf_Data *data; 1100 1101 idx++; 1102 if (gelf_getshdr(scn, &sh) != &sh) { 1103 pr_warning("failed to get section(%d) header from %s\n", 1104 idx, obj->path); 1105 err = -LIBBPF_ERRNO__FORMAT; 1106 goto out; 1107 } 1108 1109 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 1110 if (!name) { 1111 pr_warning("failed to get section(%d) name from %s\n", 1112 idx, obj->path); 1113 err = -LIBBPF_ERRNO__FORMAT; 1114 goto out; 1115 } 1116 1117 data = elf_getdata(scn, 0); 1118 if (!data) { 1119 pr_warning("failed to get section(%d) data from %s(%s)\n", 1120 idx, name, obj->path); 1121 err = -LIBBPF_ERRNO__FORMAT; 1122 goto out; 1123 } 1124 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 1125 idx, name, (unsigned long)data->d_size, 1126 (int)sh.sh_link, (unsigned long)sh.sh_flags, 1127 (int)sh.sh_type); 1128 1129 if (strcmp(name, "license") == 0) { 1130 err = bpf_object__init_license(obj, 1131 data->d_buf, 1132 data->d_size); 1133 } else if (strcmp(name, "version") == 0) { 1134 err = bpf_object__init_kversion(obj, 1135 data->d_buf, 1136 data->d_size); 1137 } else if (strcmp(name, "maps") == 0) { 1138 obj->efile.maps_shndx = idx; 1139 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 1140 btf_data = data; 1141 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 1142 btf_ext_data = data; 1143 } else if (sh.sh_type == SHT_SYMTAB) { 1144 if (obj->efile.symbols) { 1145 pr_warning("bpf: multiple SYMTAB in %s\n", 1146 obj->path); 1147 err = -LIBBPF_ERRNO__FORMAT; 1148 } else { 1149 obj->efile.symbols = data; 1150 obj->efile.strtabidx = sh.sh_link; 1151 } 1152 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 1153 if (sh.sh_flags & SHF_EXECINSTR) { 1154 if (strcmp(name, ".text") == 0) 1155 obj->efile.text_shndx = idx; 1156 err = bpf_object__add_program(obj, data->d_buf, 1157 data->d_size, name, idx); 1158 if (err) { 1159 char errmsg[STRERR_BUFSIZE]; 1160 char *cp = libbpf_strerror_r(-err, errmsg, 1161 sizeof(errmsg)); 1162 1163 pr_warning("failed to alloc program %s (%s): %s", 1164 name, obj->path, cp); 1165 } 1166 } else if (strcmp(name, ".data") == 0) { 1167 obj->efile.data = data; 1168 obj->efile.data_shndx = idx; 1169 } else if (strcmp(name, ".rodata") == 0) { 1170 obj->efile.rodata = data; 1171 obj->efile.rodata_shndx = idx; 1172 } else { 1173 pr_debug("skip section(%d) %s\n", idx, name); 1174 } 1175 } else if (sh.sh_type == SHT_REL) { 1176 void *reloc = obj->efile.reloc; 1177 int nr_reloc = obj->efile.nr_reloc + 1; 1178 int sec = sh.sh_info; /* points to other section */ 1179 1180 /* Only do relo for section with exec instructions */ 1181 if (!section_have_execinstr(obj, sec)) { 1182 pr_debug("skip relo %s(%d) for section(%d)\n", 1183 name, idx, sec); 1184 continue; 1185 } 1186 1187 reloc = reallocarray(reloc, nr_reloc, 1188 sizeof(*obj->efile.reloc)); 1189 if (!reloc) { 1190 pr_warning("realloc failed\n"); 1191 err = -ENOMEM; 1192 } else { 1193 int n = nr_reloc - 1; 1194 1195 obj->efile.reloc = reloc; 1196 obj->efile.nr_reloc = nr_reloc; 1197 1198 obj->efile.reloc[n].shdr = sh; 1199 obj->efile.reloc[n].data = data; 1200 } 1201 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { 1202 obj->efile.bss = data; 1203 obj->efile.bss_shndx = idx; 1204 } else { 1205 pr_debug("skip section(%d) %s\n", idx, name); 1206 } 1207 if (err) 1208 goto out; 1209 } 1210 1211 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 1212 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 1213 return -LIBBPF_ERRNO__FORMAT; 1214 } 1215 if (btf_data) { 1216 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 1217 if (IS_ERR(obj->btf)) { 1218 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1219 BTF_ELF_SEC, PTR_ERR(obj->btf)); 1220 obj->btf = NULL; 1221 } else { 1222 err = btf__finalize_data(obj, obj->btf); 1223 if (!err) { 1224 bpf_object__sanitize_btf(obj); 1225 err = btf__load(obj->btf); 1226 } 1227 if (err) { 1228 pr_warning("Error finalizing and loading %s into kernel: %d. Ignored and continue.\n", 1229 BTF_ELF_SEC, err); 1230 btf__free(obj->btf); 1231 obj->btf = NULL; 1232 err = 0; 1233 } 1234 } 1235 } 1236 if (btf_ext_data) { 1237 if (!obj->btf) { 1238 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 1239 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 1240 } else { 1241 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 1242 btf_ext_data->d_size); 1243 if (IS_ERR(obj->btf_ext)) { 1244 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1245 BTF_EXT_ELF_SEC, 1246 PTR_ERR(obj->btf_ext)); 1247 obj->btf_ext = NULL; 1248 } else { 1249 bpf_object__sanitize_btf_ext(obj); 1250 } 1251 } 1252 } 1253 if (bpf_object__has_maps(obj)) { 1254 err = bpf_object__init_maps(obj, flags); 1255 if (err) 1256 goto out; 1257 } 1258 err = bpf_object__init_prog_names(obj); 1259 out: 1260 return err; 1261 } 1262 1263 static struct bpf_program * 1264 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 1265 { 1266 struct bpf_program *prog; 1267 size_t i; 1268 1269 for (i = 0; i < obj->nr_programs; i++) { 1270 prog = &obj->programs[i]; 1271 if (prog->idx == idx) 1272 return prog; 1273 } 1274 return NULL; 1275 } 1276 1277 struct bpf_program * 1278 bpf_object__find_program_by_title(struct bpf_object *obj, const char *title) 1279 { 1280 struct bpf_program *pos; 1281 1282 bpf_object__for_each_program(pos, obj) { 1283 if (pos->section_name && !strcmp(pos->section_name, title)) 1284 return pos; 1285 } 1286 return NULL; 1287 } 1288 1289 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 1290 int shndx) 1291 { 1292 return shndx == obj->efile.data_shndx || 1293 shndx == obj->efile.bss_shndx || 1294 shndx == obj->efile.rodata_shndx; 1295 } 1296 1297 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 1298 int shndx) 1299 { 1300 return shndx == obj->efile.maps_shndx; 1301 } 1302 1303 static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, 1304 int shndx) 1305 { 1306 return shndx == obj->efile.text_shndx || 1307 bpf_object__shndx_is_maps(obj, shndx) || 1308 bpf_object__shndx_is_data(obj, shndx); 1309 } 1310 1311 static enum libbpf_map_type 1312 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 1313 { 1314 if (shndx == obj->efile.data_shndx) 1315 return LIBBPF_MAP_DATA; 1316 else if (shndx == obj->efile.bss_shndx) 1317 return LIBBPF_MAP_BSS; 1318 else if (shndx == obj->efile.rodata_shndx) 1319 return LIBBPF_MAP_RODATA; 1320 else 1321 return LIBBPF_MAP_UNSPEC; 1322 } 1323 1324 static int 1325 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 1326 Elf_Data *data, struct bpf_object *obj) 1327 { 1328 Elf_Data *symbols = obj->efile.symbols; 1329 struct bpf_map *maps = obj->maps; 1330 size_t nr_maps = obj->nr_maps; 1331 int i, nrels; 1332 1333 pr_debug("collecting relocating info for: '%s'\n", prog->section_name); 1334 nrels = shdr->sh_size / shdr->sh_entsize; 1335 1336 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 1337 if (!prog->reloc_desc) { 1338 pr_warning("failed to alloc memory in relocation\n"); 1339 return -ENOMEM; 1340 } 1341 prog->nr_reloc = nrels; 1342 1343 for (i = 0; i < nrels; i++) { 1344 GElf_Sym sym; 1345 GElf_Rel rel; 1346 unsigned int insn_idx; 1347 unsigned int shdr_idx; 1348 struct bpf_insn *insns = prog->insns; 1349 enum libbpf_map_type type; 1350 const char *name; 1351 size_t map_idx; 1352 1353 if (!gelf_getrel(data, i, &rel)) { 1354 pr_warning("relocation: failed to get %d reloc\n", i); 1355 return -LIBBPF_ERRNO__FORMAT; 1356 } 1357 1358 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { 1359 pr_warning("relocation: symbol %"PRIx64" not found\n", 1360 GELF_R_SYM(rel.r_info)); 1361 return -LIBBPF_ERRNO__FORMAT; 1362 } 1363 1364 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1365 sym.st_name) ? : "<?>"; 1366 1367 pr_debug("relo for %lld value %lld name %d (\'%s\')\n", 1368 (long long) (rel.r_info >> 32), 1369 (long long) sym.st_value, sym.st_name, name); 1370 1371 shdr_idx = sym.st_shndx; 1372 if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { 1373 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", 1374 prog->section_name, shdr_idx); 1375 return -LIBBPF_ERRNO__RELOC; 1376 } 1377 1378 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 1379 pr_debug("relocation: insn_idx=%u\n", insn_idx); 1380 1381 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 1382 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 1383 pr_warning("incorrect bpf_call opcode\n"); 1384 return -LIBBPF_ERRNO__RELOC; 1385 } 1386 prog->reloc_desc[i].type = RELO_CALL; 1387 prog->reloc_desc[i].insn_idx = insn_idx; 1388 prog->reloc_desc[i].text_off = sym.st_value; 1389 obj->has_pseudo_calls = true; 1390 continue; 1391 } 1392 1393 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 1394 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 1395 insn_idx, insns[insn_idx].code); 1396 return -LIBBPF_ERRNO__RELOC; 1397 } 1398 1399 if (bpf_object__shndx_is_maps(obj, shdr_idx) || 1400 bpf_object__shndx_is_data(obj, shdr_idx)) { 1401 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 1402 if (type != LIBBPF_MAP_UNSPEC) { 1403 if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { 1404 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", 1405 name, insn_idx, insns[insn_idx].code); 1406 return -LIBBPF_ERRNO__RELOC; 1407 } 1408 if (!obj->caps.global_data) { 1409 pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", 1410 name, insn_idx); 1411 return -LIBBPF_ERRNO__RELOC; 1412 } 1413 } 1414 1415 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 1416 if (maps[map_idx].libbpf_type != type) 1417 continue; 1418 if (type != LIBBPF_MAP_UNSPEC || 1419 maps[map_idx].offset == sym.st_value) { 1420 pr_debug("relocation: find map %zd (%s) for insn %u\n", 1421 map_idx, maps[map_idx].name, insn_idx); 1422 break; 1423 } 1424 } 1425 1426 if (map_idx >= nr_maps) { 1427 pr_warning("bpf relocation: map_idx %d larger than %d\n", 1428 (int)map_idx, (int)nr_maps - 1); 1429 return -LIBBPF_ERRNO__RELOC; 1430 } 1431 1432 prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? 1433 RELO_DATA : RELO_LD64; 1434 prog->reloc_desc[i].insn_idx = insn_idx; 1435 prog->reloc_desc[i].map_idx = map_idx; 1436 } 1437 } 1438 return 0; 1439 } 1440 1441 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) 1442 { 1443 struct bpf_map_def *def = &map->def; 1444 __u32 key_type_id = 0, value_type_id = 0; 1445 int ret; 1446 1447 if (!bpf_map__is_internal(map)) { 1448 ret = btf__get_map_kv_tids(btf, map->name, def->key_size, 1449 def->value_size, &key_type_id, 1450 &value_type_id); 1451 } else { 1452 /* 1453 * LLVM annotates global data differently in BTF, that is, 1454 * only as '.data', '.bss' or '.rodata'. 1455 */ 1456 ret = btf__find_by_name(btf, 1457 libbpf_type_to_btf_name[map->libbpf_type]); 1458 } 1459 if (ret < 0) 1460 return ret; 1461 1462 map->btf_key_type_id = key_type_id; 1463 map->btf_value_type_id = bpf_map__is_internal(map) ? 1464 ret : value_type_id; 1465 return 0; 1466 } 1467 1468 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 1469 { 1470 struct bpf_map_info info = {}; 1471 __u32 len = sizeof(info); 1472 int new_fd, err; 1473 char *new_name; 1474 1475 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1476 if (err) 1477 return err; 1478 1479 new_name = strdup(info.name); 1480 if (!new_name) 1481 return -errno; 1482 1483 new_fd = open("/", O_RDONLY | O_CLOEXEC); 1484 if (new_fd < 0) 1485 goto err_free_new_name; 1486 1487 new_fd = dup3(fd, new_fd, O_CLOEXEC); 1488 if (new_fd < 0) 1489 goto err_close_new_fd; 1490 1491 err = zclose(map->fd); 1492 if (err) 1493 goto err_close_new_fd; 1494 free(map->name); 1495 1496 map->fd = new_fd; 1497 map->name = new_name; 1498 map->def.type = info.type; 1499 map->def.key_size = info.key_size; 1500 map->def.value_size = info.value_size; 1501 map->def.max_entries = info.max_entries; 1502 map->def.map_flags = info.map_flags; 1503 map->btf_key_type_id = info.btf_key_type_id; 1504 map->btf_value_type_id = info.btf_value_type_id; 1505 1506 return 0; 1507 1508 err_close_new_fd: 1509 close(new_fd); 1510 err_free_new_name: 1511 free(new_name); 1512 return -errno; 1513 } 1514 1515 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 1516 { 1517 if (!map || !max_entries) 1518 return -EINVAL; 1519 1520 /* If map already created, its attributes can't be changed. */ 1521 if (map->fd >= 0) 1522 return -EBUSY; 1523 1524 map->def.max_entries = max_entries; 1525 1526 return 0; 1527 } 1528 1529 static int 1530 bpf_object__probe_name(struct bpf_object *obj) 1531 { 1532 struct bpf_load_program_attr attr; 1533 char *cp, errmsg[STRERR_BUFSIZE]; 1534 struct bpf_insn insns[] = { 1535 BPF_MOV64_IMM(BPF_REG_0, 0), 1536 BPF_EXIT_INSN(), 1537 }; 1538 int ret; 1539 1540 /* make sure basic loading works */ 1541 1542 memset(&attr, 0, sizeof(attr)); 1543 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1544 attr.insns = insns; 1545 attr.insns_cnt = ARRAY_SIZE(insns); 1546 attr.license = "GPL"; 1547 1548 ret = bpf_load_program_xattr(&attr, NULL, 0); 1549 if (ret < 0) { 1550 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1551 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", 1552 __func__, cp, errno); 1553 return -errno; 1554 } 1555 close(ret); 1556 1557 /* now try the same program, but with the name */ 1558 1559 attr.name = "test"; 1560 ret = bpf_load_program_xattr(&attr, NULL, 0); 1561 if (ret >= 0) { 1562 obj->caps.name = 1; 1563 close(ret); 1564 } 1565 1566 return 0; 1567 } 1568 1569 static int 1570 bpf_object__probe_global_data(struct bpf_object *obj) 1571 { 1572 struct bpf_load_program_attr prg_attr; 1573 struct bpf_create_map_attr map_attr; 1574 char *cp, errmsg[STRERR_BUFSIZE]; 1575 struct bpf_insn insns[] = { 1576 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 1577 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 1578 BPF_MOV64_IMM(BPF_REG_0, 0), 1579 BPF_EXIT_INSN(), 1580 }; 1581 int ret, map; 1582 1583 memset(&map_attr, 0, sizeof(map_attr)); 1584 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 1585 map_attr.key_size = sizeof(int); 1586 map_attr.value_size = 32; 1587 map_attr.max_entries = 1; 1588 1589 map = bpf_create_map_xattr(&map_attr); 1590 if (map < 0) { 1591 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1592 pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", 1593 __func__, cp, errno); 1594 return -errno; 1595 } 1596 1597 insns[0].imm = map; 1598 1599 memset(&prg_attr, 0, sizeof(prg_attr)); 1600 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1601 prg_attr.insns = insns; 1602 prg_attr.insns_cnt = ARRAY_SIZE(insns); 1603 prg_attr.license = "GPL"; 1604 1605 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 1606 if (ret >= 0) { 1607 obj->caps.global_data = 1; 1608 close(ret); 1609 } 1610 1611 close(map); 1612 return 0; 1613 } 1614 1615 static int bpf_object__probe_btf_func(struct bpf_object *obj) 1616 { 1617 const char strs[] = "\0int\0x\0a"; 1618 /* void x(int a) {} */ 1619 __u32 types[] = { 1620 /* int */ 1621 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 1622 /* FUNC_PROTO */ /* [2] */ 1623 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 1624 BTF_PARAM_ENC(7, 1), 1625 /* FUNC x */ /* [3] */ 1626 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 1627 }; 1628 int res; 1629 1630 res = libbpf__probe_raw_btf((char *)types, sizeof(types), 1631 strs, sizeof(strs)); 1632 if (res < 0) 1633 return res; 1634 if (res > 0) 1635 obj->caps.btf_func = 1; 1636 return 0; 1637 } 1638 1639 static int bpf_object__probe_btf_datasec(struct bpf_object *obj) 1640 { 1641 const char strs[] = "\0x\0.data"; 1642 /* static int a; */ 1643 __u32 types[] = { 1644 /* int */ 1645 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 1646 /* VAR x */ /* [2] */ 1647 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 1648 BTF_VAR_STATIC, 1649 /* DATASEC val */ /* [3] */ 1650 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 1651 BTF_VAR_SECINFO_ENC(2, 0, 4), 1652 }; 1653 int res; 1654 1655 res = libbpf__probe_raw_btf((char *)types, sizeof(types), 1656 strs, sizeof(strs)); 1657 if (res < 0) 1658 return res; 1659 if (res > 0) 1660 obj->caps.btf_datasec = 1; 1661 return 0; 1662 } 1663 1664 static int 1665 bpf_object__probe_caps(struct bpf_object *obj) 1666 { 1667 int (*probe_fn[])(struct bpf_object *obj) = { 1668 bpf_object__probe_name, 1669 bpf_object__probe_global_data, 1670 bpf_object__probe_btf_func, 1671 bpf_object__probe_btf_datasec, 1672 }; 1673 int i, ret; 1674 1675 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { 1676 ret = probe_fn[i](obj); 1677 if (ret < 0) 1678 pr_debug("Probe #%d failed with %d.\n", i, ret); 1679 } 1680 1681 return 0; 1682 } 1683 1684 static int 1685 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 1686 { 1687 char *cp, errmsg[STRERR_BUFSIZE]; 1688 int err, zero = 0; 1689 __u8 *data; 1690 1691 /* Nothing to do here since kernel already zero-initializes .bss map. */ 1692 if (map->libbpf_type == LIBBPF_MAP_BSS) 1693 return 0; 1694 1695 data = map->libbpf_type == LIBBPF_MAP_DATA ? 1696 obj->sections.data : obj->sections.rodata; 1697 1698 err = bpf_map_update_elem(map->fd, &zero, data, 0); 1699 /* Freeze .rodata map as read-only from syscall side. */ 1700 if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { 1701 err = bpf_map_freeze(map->fd); 1702 if (err) { 1703 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1704 pr_warning("Error freezing map(%s) as read-only: %s\n", 1705 map->name, cp); 1706 err = 0; 1707 } 1708 } 1709 return err; 1710 } 1711 1712 static int 1713 bpf_object__create_maps(struct bpf_object *obj) 1714 { 1715 struct bpf_create_map_attr create_attr = {}; 1716 unsigned int i; 1717 int err; 1718 1719 for (i = 0; i < obj->nr_maps; i++) { 1720 struct bpf_map *map = &obj->maps[i]; 1721 struct bpf_map_def *def = &map->def; 1722 char *cp, errmsg[STRERR_BUFSIZE]; 1723 int *pfd = &map->fd; 1724 1725 if (map->fd >= 0) { 1726 pr_debug("skip map create (preset) %s: fd=%d\n", 1727 map->name, map->fd); 1728 continue; 1729 } 1730 1731 if (obj->caps.name) 1732 create_attr.name = map->name; 1733 create_attr.map_ifindex = map->map_ifindex; 1734 create_attr.map_type = def->type; 1735 create_attr.map_flags = def->map_flags; 1736 create_attr.key_size = def->key_size; 1737 create_attr.value_size = def->value_size; 1738 create_attr.max_entries = def->max_entries; 1739 create_attr.btf_fd = -1; 1740 create_attr.btf_key_type_id = 0; 1741 create_attr.btf_value_type_id = 0; 1742 if (bpf_map_type__is_map_in_map(def->type) && 1743 map->inner_map_fd >= 0) 1744 create_attr.inner_map_fd = map->inner_map_fd; 1745 1746 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) { 1747 create_attr.btf_fd = btf__fd(obj->btf); 1748 create_attr.btf_key_type_id = map->btf_key_type_id; 1749 create_attr.btf_value_type_id = map->btf_value_type_id; 1750 } 1751 1752 *pfd = bpf_create_map_xattr(&create_attr); 1753 if (*pfd < 0 && create_attr.btf_fd >= 0) { 1754 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1755 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 1756 map->name, cp, errno); 1757 create_attr.btf_fd = -1; 1758 create_attr.btf_key_type_id = 0; 1759 create_attr.btf_value_type_id = 0; 1760 map->btf_key_type_id = 0; 1761 map->btf_value_type_id = 0; 1762 *pfd = bpf_create_map_xattr(&create_attr); 1763 } 1764 1765 if (*pfd < 0) { 1766 size_t j; 1767 1768 err = *pfd; 1769 err_out: 1770 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1771 pr_warning("failed to create map (name: '%s'): %s\n", 1772 map->name, cp); 1773 for (j = 0; j < i; j++) 1774 zclose(obj->maps[j].fd); 1775 return err; 1776 } 1777 1778 if (bpf_map__is_internal(map)) { 1779 err = bpf_object__populate_internal_map(obj, map); 1780 if (err < 0) { 1781 zclose(*pfd); 1782 goto err_out; 1783 } 1784 } 1785 1786 pr_debug("created map %s: fd=%d\n", map->name, *pfd); 1787 } 1788 1789 return 0; 1790 } 1791 1792 static int 1793 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 1794 void *btf_prog_info, const char *info_name) 1795 { 1796 if (err != -ENOENT) { 1797 pr_warning("Error in loading %s for sec %s.\n", 1798 info_name, prog->section_name); 1799 return err; 1800 } 1801 1802 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 1803 1804 if (btf_prog_info) { 1805 /* 1806 * Some info has already been found but has problem 1807 * in the last btf_ext reloc. Must have to error out. 1808 */ 1809 pr_warning("Error in relocating %s for sec %s.\n", 1810 info_name, prog->section_name); 1811 return err; 1812 } 1813 1814 /* Have problem loading the very first info. Ignore the rest. */ 1815 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", 1816 info_name, prog->section_name, info_name); 1817 return 0; 1818 } 1819 1820 static int 1821 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 1822 const char *section_name, __u32 insn_offset) 1823 { 1824 int err; 1825 1826 if (!insn_offset || prog->func_info) { 1827 /* 1828 * !insn_offset => main program 1829 * 1830 * For sub prog, the main program's func_info has to 1831 * be loaded first (i.e. prog->func_info != NULL) 1832 */ 1833 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 1834 section_name, insn_offset, 1835 &prog->func_info, 1836 &prog->func_info_cnt); 1837 if (err) 1838 return check_btf_ext_reloc_err(prog, err, 1839 prog->func_info, 1840 "bpf_func_info"); 1841 1842 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 1843 } 1844 1845 if (!insn_offset || prog->line_info) { 1846 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 1847 section_name, insn_offset, 1848 &prog->line_info, 1849 &prog->line_info_cnt); 1850 if (err) 1851 return check_btf_ext_reloc_err(prog, err, 1852 prog->line_info, 1853 "bpf_line_info"); 1854 1855 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 1856 } 1857 1858 if (!insn_offset) 1859 prog->btf_fd = btf__fd(obj->btf); 1860 1861 return 0; 1862 } 1863 1864 static int 1865 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 1866 struct reloc_desc *relo) 1867 { 1868 struct bpf_insn *insn, *new_insn; 1869 struct bpf_program *text; 1870 size_t new_cnt; 1871 int err; 1872 1873 if (relo->type != RELO_CALL) 1874 return -LIBBPF_ERRNO__RELOC; 1875 1876 if (prog->idx == obj->efile.text_shndx) { 1877 pr_warning("relo in .text insn %d into off %d\n", 1878 relo->insn_idx, relo->text_off); 1879 return -LIBBPF_ERRNO__RELOC; 1880 } 1881 1882 if (prog->main_prog_cnt == 0) { 1883 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 1884 if (!text) { 1885 pr_warning("no .text section found yet relo into text exist\n"); 1886 return -LIBBPF_ERRNO__RELOC; 1887 } 1888 new_cnt = prog->insns_cnt + text->insns_cnt; 1889 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 1890 if (!new_insn) { 1891 pr_warning("oom in prog realloc\n"); 1892 return -ENOMEM; 1893 } 1894 1895 if (obj->btf_ext) { 1896 err = bpf_program_reloc_btf_ext(prog, obj, 1897 text->section_name, 1898 prog->insns_cnt); 1899 if (err) 1900 return err; 1901 } 1902 1903 memcpy(new_insn + prog->insns_cnt, text->insns, 1904 text->insns_cnt * sizeof(*insn)); 1905 prog->insns = new_insn; 1906 prog->main_prog_cnt = prog->insns_cnt; 1907 prog->insns_cnt = new_cnt; 1908 pr_debug("added %zd insn from %s to prog %s\n", 1909 text->insns_cnt, text->section_name, 1910 prog->section_name); 1911 } 1912 insn = &prog->insns[relo->insn_idx]; 1913 insn->imm += prog->main_prog_cnt - relo->insn_idx; 1914 return 0; 1915 } 1916 1917 static int 1918 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1919 { 1920 int i, err; 1921 1922 if (!prog) 1923 return 0; 1924 1925 if (obj->btf_ext) { 1926 err = bpf_program_reloc_btf_ext(prog, obj, 1927 prog->section_name, 0); 1928 if (err) 1929 return err; 1930 } 1931 1932 if (!prog->reloc_desc) 1933 return 0; 1934 1935 for (i = 0; i < prog->nr_reloc; i++) { 1936 if (prog->reloc_desc[i].type == RELO_LD64 || 1937 prog->reloc_desc[i].type == RELO_DATA) { 1938 bool relo_data = prog->reloc_desc[i].type == RELO_DATA; 1939 struct bpf_insn *insns = prog->insns; 1940 int insn_idx, map_idx; 1941 1942 insn_idx = prog->reloc_desc[i].insn_idx; 1943 map_idx = prog->reloc_desc[i].map_idx; 1944 1945 if (insn_idx + 1 >= (int)prog->insns_cnt) { 1946 pr_warning("relocation out of range: '%s'\n", 1947 prog->section_name); 1948 return -LIBBPF_ERRNO__RELOC; 1949 } 1950 1951 if (!relo_data) { 1952 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1953 } else { 1954 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; 1955 insns[insn_idx + 1].imm = insns[insn_idx].imm; 1956 } 1957 insns[insn_idx].imm = obj->maps[map_idx].fd; 1958 } else if (prog->reloc_desc[i].type == RELO_CALL) { 1959 err = bpf_program__reloc_text(prog, obj, 1960 &prog->reloc_desc[i]); 1961 if (err) 1962 return err; 1963 } 1964 } 1965 1966 zfree(&prog->reloc_desc); 1967 prog->nr_reloc = 0; 1968 return 0; 1969 } 1970 1971 1972 static int 1973 bpf_object__relocate(struct bpf_object *obj) 1974 { 1975 struct bpf_program *prog; 1976 size_t i; 1977 int err; 1978 1979 for (i = 0; i < obj->nr_programs; i++) { 1980 prog = &obj->programs[i]; 1981 1982 err = bpf_program__relocate(prog, obj); 1983 if (err) { 1984 pr_warning("failed to relocate '%s'\n", 1985 prog->section_name); 1986 return err; 1987 } 1988 } 1989 return 0; 1990 } 1991 1992 static int bpf_object__collect_reloc(struct bpf_object *obj) 1993 { 1994 int i, err; 1995 1996 if (!obj_elf_valid(obj)) { 1997 pr_warning("Internal error: elf object is closed\n"); 1998 return -LIBBPF_ERRNO__INTERNAL; 1999 } 2000 2001 for (i = 0; i < obj->efile.nr_reloc; i++) { 2002 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 2003 Elf_Data *data = obj->efile.reloc[i].data; 2004 int idx = shdr->sh_info; 2005 struct bpf_program *prog; 2006 2007 if (shdr->sh_type != SHT_REL) { 2008 pr_warning("internal error at %d\n", __LINE__); 2009 return -LIBBPF_ERRNO__INTERNAL; 2010 } 2011 2012 prog = bpf_object__find_prog_by_idx(obj, idx); 2013 if (!prog) { 2014 pr_warning("relocation failed: no section(%d)\n", idx); 2015 return -LIBBPF_ERRNO__RELOC; 2016 } 2017 2018 err = bpf_program__collect_reloc(prog, shdr, data, obj); 2019 if (err) 2020 return err; 2021 } 2022 return 0; 2023 } 2024 2025 static int 2026 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 2027 char *license, __u32 kern_version, int *pfd) 2028 { 2029 struct bpf_load_program_attr load_attr; 2030 char *cp, errmsg[STRERR_BUFSIZE]; 2031 int log_buf_size = BPF_LOG_BUF_SIZE; 2032 char *log_buf; 2033 int ret; 2034 2035 if (!insns || !insns_cnt) 2036 return -EINVAL; 2037 2038 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 2039 load_attr.prog_type = prog->type; 2040 load_attr.expected_attach_type = prog->expected_attach_type; 2041 if (prog->caps->name) 2042 load_attr.name = prog->name; 2043 load_attr.insns = insns; 2044 load_attr.insns_cnt = insns_cnt; 2045 load_attr.license = license; 2046 load_attr.kern_version = kern_version; 2047 load_attr.prog_ifindex = prog->prog_ifindex; 2048 load_attr.prog_btf_fd = prog->btf_fd; 2049 load_attr.func_info = prog->func_info; 2050 load_attr.func_info_rec_size = prog->func_info_rec_size; 2051 load_attr.func_info_cnt = prog->func_info_cnt; 2052 load_attr.line_info = prog->line_info; 2053 load_attr.line_info_rec_size = prog->line_info_rec_size; 2054 load_attr.line_info_cnt = prog->line_info_cnt; 2055 load_attr.log_level = prog->log_level; 2056 load_attr.prog_flags = prog->prog_flags; 2057 2058 retry_load: 2059 log_buf = malloc(log_buf_size); 2060 if (!log_buf) 2061 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 2062 2063 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); 2064 2065 if (ret >= 0) { 2066 if (load_attr.log_level) 2067 pr_debug("verifier log:\n%s", log_buf); 2068 *pfd = ret; 2069 ret = 0; 2070 goto out; 2071 } 2072 2073 if (errno == ENOSPC) { 2074 log_buf_size <<= 1; 2075 free(log_buf); 2076 goto retry_load; 2077 } 2078 ret = -LIBBPF_ERRNO__LOAD; 2079 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2080 pr_warning("load bpf program failed: %s\n", cp); 2081 2082 if (log_buf && log_buf[0] != '\0') { 2083 ret = -LIBBPF_ERRNO__VERIFY; 2084 pr_warning("-- BEGIN DUMP LOG ---\n"); 2085 pr_warning("\n%s\n", log_buf); 2086 pr_warning("-- END LOG --\n"); 2087 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 2088 pr_warning("Program too large (%zu insns), at most %d insns\n", 2089 load_attr.insns_cnt, BPF_MAXINSNS); 2090 ret = -LIBBPF_ERRNO__PROG2BIG; 2091 } else { 2092 /* Wrong program type? */ 2093 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 2094 int fd; 2095 2096 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 2097 load_attr.expected_attach_type = 0; 2098 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 2099 if (fd >= 0) { 2100 close(fd); 2101 ret = -LIBBPF_ERRNO__PROGTYPE; 2102 goto out; 2103 } 2104 } 2105 2106 if (log_buf) 2107 ret = -LIBBPF_ERRNO__KVER; 2108 } 2109 2110 out: 2111 free(log_buf); 2112 return ret; 2113 } 2114 2115 int 2116 bpf_program__load(struct bpf_program *prog, 2117 char *license, __u32 kern_version) 2118 { 2119 int err = 0, fd, i; 2120 2121 if (prog->instances.nr < 0 || !prog->instances.fds) { 2122 if (prog->preprocessor) { 2123 pr_warning("Internal error: can't load program '%s'\n", 2124 prog->section_name); 2125 return -LIBBPF_ERRNO__INTERNAL; 2126 } 2127 2128 prog->instances.fds = malloc(sizeof(int)); 2129 if (!prog->instances.fds) { 2130 pr_warning("Not enough memory for BPF fds\n"); 2131 return -ENOMEM; 2132 } 2133 prog->instances.nr = 1; 2134 prog->instances.fds[0] = -1; 2135 } 2136 2137 if (!prog->preprocessor) { 2138 if (prog->instances.nr != 1) { 2139 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 2140 prog->section_name, prog->instances.nr); 2141 } 2142 err = load_program(prog, prog->insns, prog->insns_cnt, 2143 license, kern_version, &fd); 2144 if (!err) 2145 prog->instances.fds[0] = fd; 2146 goto out; 2147 } 2148 2149 for (i = 0; i < prog->instances.nr; i++) { 2150 struct bpf_prog_prep_result result; 2151 bpf_program_prep_t preprocessor = prog->preprocessor; 2152 2153 memset(&result, 0, sizeof(result)); 2154 err = preprocessor(prog, i, prog->insns, 2155 prog->insns_cnt, &result); 2156 if (err) { 2157 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 2158 i, prog->section_name); 2159 goto out; 2160 } 2161 2162 if (!result.new_insn_ptr || !result.new_insn_cnt) { 2163 pr_debug("Skip loading the %dth instance of program '%s'\n", 2164 i, prog->section_name); 2165 prog->instances.fds[i] = -1; 2166 if (result.pfd) 2167 *result.pfd = -1; 2168 continue; 2169 } 2170 2171 err = load_program(prog, result.new_insn_ptr, 2172 result.new_insn_cnt, 2173 license, kern_version, &fd); 2174 2175 if (err) { 2176 pr_warning("Loading the %dth instance of program '%s' failed\n", 2177 i, prog->section_name); 2178 goto out; 2179 } 2180 2181 if (result.pfd) 2182 *result.pfd = fd; 2183 prog->instances.fds[i] = fd; 2184 } 2185 out: 2186 if (err) 2187 pr_warning("failed to load program '%s'\n", 2188 prog->section_name); 2189 zfree(&prog->insns); 2190 prog->insns_cnt = 0; 2191 return err; 2192 } 2193 2194 static bool bpf_program__is_function_storage(struct bpf_program *prog, 2195 struct bpf_object *obj) 2196 { 2197 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 2198 } 2199 2200 static int 2201 bpf_object__load_progs(struct bpf_object *obj, int log_level) 2202 { 2203 size_t i; 2204 int err; 2205 2206 for (i = 0; i < obj->nr_programs; i++) { 2207 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 2208 continue; 2209 obj->programs[i].log_level |= log_level; 2210 err = bpf_program__load(&obj->programs[i], 2211 obj->license, 2212 obj->kern_version); 2213 if (err) 2214 return err; 2215 } 2216 return 0; 2217 } 2218 2219 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) 2220 { 2221 switch (type) { 2222 case BPF_PROG_TYPE_SOCKET_FILTER: 2223 case BPF_PROG_TYPE_SCHED_CLS: 2224 case BPF_PROG_TYPE_SCHED_ACT: 2225 case BPF_PROG_TYPE_XDP: 2226 case BPF_PROG_TYPE_CGROUP_SKB: 2227 case BPF_PROG_TYPE_CGROUP_SOCK: 2228 case BPF_PROG_TYPE_LWT_IN: 2229 case BPF_PROG_TYPE_LWT_OUT: 2230 case BPF_PROG_TYPE_LWT_XMIT: 2231 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 2232 case BPF_PROG_TYPE_SOCK_OPS: 2233 case BPF_PROG_TYPE_SK_SKB: 2234 case BPF_PROG_TYPE_CGROUP_DEVICE: 2235 case BPF_PROG_TYPE_SK_MSG: 2236 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2237 case BPF_PROG_TYPE_LIRC_MODE2: 2238 case BPF_PROG_TYPE_SK_REUSEPORT: 2239 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2240 case BPF_PROG_TYPE_UNSPEC: 2241 case BPF_PROG_TYPE_TRACEPOINT: 2242 case BPF_PROG_TYPE_RAW_TRACEPOINT: 2243 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2244 case BPF_PROG_TYPE_PERF_EVENT: 2245 case BPF_PROG_TYPE_CGROUP_SYSCTL: 2246 return false; 2247 case BPF_PROG_TYPE_KPROBE: 2248 default: 2249 return true; 2250 } 2251 } 2252 2253 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) 2254 { 2255 if (needs_kver && obj->kern_version == 0) { 2256 pr_warning("%s doesn't provide kernel version\n", 2257 obj->path); 2258 return -LIBBPF_ERRNO__KVERSION; 2259 } 2260 return 0; 2261 } 2262 2263 static struct bpf_object * 2264 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, 2265 bool needs_kver, int flags) 2266 { 2267 struct bpf_object *obj; 2268 int err; 2269 2270 if (elf_version(EV_CURRENT) == EV_NONE) { 2271 pr_warning("failed to init libelf for %s\n", path); 2272 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 2273 } 2274 2275 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 2276 if (IS_ERR(obj)) 2277 return obj; 2278 2279 CHECK_ERR(bpf_object__elf_init(obj), err, out); 2280 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 2281 CHECK_ERR(bpf_object__probe_caps(obj), err, out); 2282 CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); 2283 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 2284 CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); 2285 2286 bpf_object__elf_finish(obj); 2287 return obj; 2288 out: 2289 bpf_object__close(obj); 2290 return ERR_PTR(err); 2291 } 2292 2293 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, 2294 int flags) 2295 { 2296 /* param validation */ 2297 if (!attr->file) 2298 return NULL; 2299 2300 pr_debug("loading %s\n", attr->file); 2301 2302 return __bpf_object__open(attr->file, NULL, 0, 2303 bpf_prog_type__needs_kver(attr->prog_type), 2304 flags); 2305 } 2306 2307 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 2308 { 2309 return __bpf_object__open_xattr(attr, 0); 2310 } 2311 2312 struct bpf_object *bpf_object__open(const char *path) 2313 { 2314 struct bpf_object_open_attr attr = { 2315 .file = path, 2316 .prog_type = BPF_PROG_TYPE_UNSPEC, 2317 }; 2318 2319 return bpf_object__open_xattr(&attr); 2320 } 2321 2322 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 2323 size_t obj_buf_sz, 2324 const char *name) 2325 { 2326 char tmp_name[64]; 2327 2328 /* param validation */ 2329 if (!obj_buf || obj_buf_sz <= 0) 2330 return NULL; 2331 2332 if (!name) { 2333 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 2334 (unsigned long)obj_buf, 2335 (unsigned long)obj_buf_sz); 2336 name = tmp_name; 2337 } 2338 pr_debug("loading object '%s' from buffer\n", name); 2339 2340 return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); 2341 } 2342 2343 int bpf_object__unload(struct bpf_object *obj) 2344 { 2345 size_t i; 2346 2347 if (!obj) 2348 return -EINVAL; 2349 2350 for (i = 0; i < obj->nr_maps; i++) 2351 zclose(obj->maps[i].fd); 2352 2353 for (i = 0; i < obj->nr_programs; i++) 2354 bpf_program__unload(&obj->programs[i]); 2355 2356 return 0; 2357 } 2358 2359 int bpf_object__load_xattr(struct bpf_object_load_attr *attr) 2360 { 2361 struct bpf_object *obj; 2362 int err; 2363 2364 if (!attr) 2365 return -EINVAL; 2366 obj = attr->obj; 2367 if (!obj) 2368 return -EINVAL; 2369 2370 if (obj->loaded) { 2371 pr_warning("object should not be loaded twice\n"); 2372 return -EINVAL; 2373 } 2374 2375 obj->loaded = true; 2376 2377 CHECK_ERR(bpf_object__create_maps(obj), err, out); 2378 CHECK_ERR(bpf_object__relocate(obj), err, out); 2379 CHECK_ERR(bpf_object__load_progs(obj, attr->log_level), err, out); 2380 2381 return 0; 2382 out: 2383 bpf_object__unload(obj); 2384 pr_warning("failed to load object '%s'\n", obj->path); 2385 return err; 2386 } 2387 2388 int bpf_object__load(struct bpf_object *obj) 2389 { 2390 struct bpf_object_load_attr attr = { 2391 .obj = obj, 2392 }; 2393 2394 return bpf_object__load_xattr(&attr); 2395 } 2396 2397 static int check_path(const char *path) 2398 { 2399 char *cp, errmsg[STRERR_BUFSIZE]; 2400 struct statfs st_fs; 2401 char *dname, *dir; 2402 int err = 0; 2403 2404 if (path == NULL) 2405 return -EINVAL; 2406 2407 dname = strdup(path); 2408 if (dname == NULL) 2409 return -ENOMEM; 2410 2411 dir = dirname(dname); 2412 if (statfs(dir, &st_fs)) { 2413 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2414 pr_warning("failed to statfs %s: %s\n", dir, cp); 2415 err = -errno; 2416 } 2417 free(dname); 2418 2419 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 2420 pr_warning("specified path %s is not on BPF FS\n", path); 2421 err = -EINVAL; 2422 } 2423 2424 return err; 2425 } 2426 2427 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 2428 int instance) 2429 { 2430 char *cp, errmsg[STRERR_BUFSIZE]; 2431 int err; 2432 2433 err = check_path(path); 2434 if (err) 2435 return err; 2436 2437 if (prog == NULL) { 2438 pr_warning("invalid program pointer\n"); 2439 return -EINVAL; 2440 } 2441 2442 if (instance < 0 || instance >= prog->instances.nr) { 2443 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2444 instance, prog->section_name, prog->instances.nr); 2445 return -EINVAL; 2446 } 2447 2448 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 2449 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2450 pr_warning("failed to pin program: %s\n", cp); 2451 return -errno; 2452 } 2453 pr_debug("pinned program '%s'\n", path); 2454 2455 return 0; 2456 } 2457 2458 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 2459 int instance) 2460 { 2461 int err; 2462 2463 err = check_path(path); 2464 if (err) 2465 return err; 2466 2467 if (prog == NULL) { 2468 pr_warning("invalid program pointer\n"); 2469 return -EINVAL; 2470 } 2471 2472 if (instance < 0 || instance >= prog->instances.nr) { 2473 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2474 instance, prog->section_name, prog->instances.nr); 2475 return -EINVAL; 2476 } 2477 2478 err = unlink(path); 2479 if (err != 0) 2480 return -errno; 2481 pr_debug("unpinned program '%s'\n", path); 2482 2483 return 0; 2484 } 2485 2486 static int make_dir(const char *path) 2487 { 2488 char *cp, errmsg[STRERR_BUFSIZE]; 2489 int err = 0; 2490 2491 if (mkdir(path, 0700) && errno != EEXIST) 2492 err = -errno; 2493 2494 if (err) { 2495 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 2496 pr_warning("failed to mkdir %s: %s\n", path, cp); 2497 } 2498 return err; 2499 } 2500 2501 int bpf_program__pin(struct bpf_program *prog, const char *path) 2502 { 2503 int i, err; 2504 2505 err = check_path(path); 2506 if (err) 2507 return err; 2508 2509 if (prog == NULL) { 2510 pr_warning("invalid program pointer\n"); 2511 return -EINVAL; 2512 } 2513 2514 if (prog->instances.nr <= 0) { 2515 pr_warning("no instances of prog %s to pin\n", 2516 prog->section_name); 2517 return -EINVAL; 2518 } 2519 2520 if (prog->instances.nr == 1) { 2521 /* don't create subdirs when pinning single instance */ 2522 return bpf_program__pin_instance(prog, path, 0); 2523 } 2524 2525 err = make_dir(path); 2526 if (err) 2527 return err; 2528 2529 for (i = 0; i < prog->instances.nr; i++) { 2530 char buf[PATH_MAX]; 2531 int len; 2532 2533 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2534 if (len < 0) { 2535 err = -EINVAL; 2536 goto err_unpin; 2537 } else if (len >= PATH_MAX) { 2538 err = -ENAMETOOLONG; 2539 goto err_unpin; 2540 } 2541 2542 err = bpf_program__pin_instance(prog, buf, i); 2543 if (err) 2544 goto err_unpin; 2545 } 2546 2547 return 0; 2548 2549 err_unpin: 2550 for (i = i - 1; i >= 0; i--) { 2551 char buf[PATH_MAX]; 2552 int len; 2553 2554 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2555 if (len < 0) 2556 continue; 2557 else if (len >= PATH_MAX) 2558 continue; 2559 2560 bpf_program__unpin_instance(prog, buf, i); 2561 } 2562 2563 rmdir(path); 2564 2565 return err; 2566 } 2567 2568 int bpf_program__unpin(struct bpf_program *prog, const char *path) 2569 { 2570 int i, err; 2571 2572 err = check_path(path); 2573 if (err) 2574 return err; 2575 2576 if (prog == NULL) { 2577 pr_warning("invalid program pointer\n"); 2578 return -EINVAL; 2579 } 2580 2581 if (prog->instances.nr <= 0) { 2582 pr_warning("no instances of prog %s to pin\n", 2583 prog->section_name); 2584 return -EINVAL; 2585 } 2586 2587 if (prog->instances.nr == 1) { 2588 /* don't create subdirs when pinning single instance */ 2589 return bpf_program__unpin_instance(prog, path, 0); 2590 } 2591 2592 for (i = 0; i < prog->instances.nr; i++) { 2593 char buf[PATH_MAX]; 2594 int len; 2595 2596 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2597 if (len < 0) 2598 return -EINVAL; 2599 else if (len >= PATH_MAX) 2600 return -ENAMETOOLONG; 2601 2602 err = bpf_program__unpin_instance(prog, buf, i); 2603 if (err) 2604 return err; 2605 } 2606 2607 err = rmdir(path); 2608 if (err) 2609 return -errno; 2610 2611 return 0; 2612 } 2613 2614 int bpf_map__pin(struct bpf_map *map, const char *path) 2615 { 2616 char *cp, errmsg[STRERR_BUFSIZE]; 2617 int err; 2618 2619 err = check_path(path); 2620 if (err) 2621 return err; 2622 2623 if (map == NULL) { 2624 pr_warning("invalid map pointer\n"); 2625 return -EINVAL; 2626 } 2627 2628 if (bpf_obj_pin(map->fd, path)) { 2629 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2630 pr_warning("failed to pin map: %s\n", cp); 2631 return -errno; 2632 } 2633 2634 pr_debug("pinned map '%s'\n", path); 2635 2636 return 0; 2637 } 2638 2639 int bpf_map__unpin(struct bpf_map *map, const char *path) 2640 { 2641 int err; 2642 2643 err = check_path(path); 2644 if (err) 2645 return err; 2646 2647 if (map == NULL) { 2648 pr_warning("invalid map pointer\n"); 2649 return -EINVAL; 2650 } 2651 2652 err = unlink(path); 2653 if (err != 0) 2654 return -errno; 2655 pr_debug("unpinned map '%s'\n", path); 2656 2657 return 0; 2658 } 2659 2660 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 2661 { 2662 struct bpf_map *map; 2663 int err; 2664 2665 if (!obj) 2666 return -ENOENT; 2667 2668 if (!obj->loaded) { 2669 pr_warning("object not yet loaded; load it first\n"); 2670 return -ENOENT; 2671 } 2672 2673 err = make_dir(path); 2674 if (err) 2675 return err; 2676 2677 bpf_object__for_each_map(map, obj) { 2678 char buf[PATH_MAX]; 2679 int len; 2680 2681 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2682 bpf_map__name(map)); 2683 if (len < 0) { 2684 err = -EINVAL; 2685 goto err_unpin_maps; 2686 } else if (len >= PATH_MAX) { 2687 err = -ENAMETOOLONG; 2688 goto err_unpin_maps; 2689 } 2690 2691 err = bpf_map__pin(map, buf); 2692 if (err) 2693 goto err_unpin_maps; 2694 } 2695 2696 return 0; 2697 2698 err_unpin_maps: 2699 while ((map = bpf_map__prev(map, obj))) { 2700 char buf[PATH_MAX]; 2701 int len; 2702 2703 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2704 bpf_map__name(map)); 2705 if (len < 0) 2706 continue; 2707 else if (len >= PATH_MAX) 2708 continue; 2709 2710 bpf_map__unpin(map, buf); 2711 } 2712 2713 return err; 2714 } 2715 2716 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 2717 { 2718 struct bpf_map *map; 2719 int err; 2720 2721 if (!obj) 2722 return -ENOENT; 2723 2724 bpf_object__for_each_map(map, obj) { 2725 char buf[PATH_MAX]; 2726 int len; 2727 2728 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2729 bpf_map__name(map)); 2730 if (len < 0) 2731 return -EINVAL; 2732 else if (len >= PATH_MAX) 2733 return -ENAMETOOLONG; 2734 2735 err = bpf_map__unpin(map, buf); 2736 if (err) 2737 return err; 2738 } 2739 2740 return 0; 2741 } 2742 2743 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 2744 { 2745 struct bpf_program *prog; 2746 int err; 2747 2748 if (!obj) 2749 return -ENOENT; 2750 2751 if (!obj->loaded) { 2752 pr_warning("object not yet loaded; load it first\n"); 2753 return -ENOENT; 2754 } 2755 2756 err = make_dir(path); 2757 if (err) 2758 return err; 2759 2760 bpf_object__for_each_program(prog, obj) { 2761 char buf[PATH_MAX]; 2762 int len; 2763 2764 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2765 prog->pin_name); 2766 if (len < 0) { 2767 err = -EINVAL; 2768 goto err_unpin_programs; 2769 } else if (len >= PATH_MAX) { 2770 err = -ENAMETOOLONG; 2771 goto err_unpin_programs; 2772 } 2773 2774 err = bpf_program__pin(prog, buf); 2775 if (err) 2776 goto err_unpin_programs; 2777 } 2778 2779 return 0; 2780 2781 err_unpin_programs: 2782 while ((prog = bpf_program__prev(prog, obj))) { 2783 char buf[PATH_MAX]; 2784 int len; 2785 2786 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2787 prog->pin_name); 2788 if (len < 0) 2789 continue; 2790 else if (len >= PATH_MAX) 2791 continue; 2792 2793 bpf_program__unpin(prog, buf); 2794 } 2795 2796 return err; 2797 } 2798 2799 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 2800 { 2801 struct bpf_program *prog; 2802 int err; 2803 2804 if (!obj) 2805 return -ENOENT; 2806 2807 bpf_object__for_each_program(prog, obj) { 2808 char buf[PATH_MAX]; 2809 int len; 2810 2811 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2812 prog->pin_name); 2813 if (len < 0) 2814 return -EINVAL; 2815 else if (len >= PATH_MAX) 2816 return -ENAMETOOLONG; 2817 2818 err = bpf_program__unpin(prog, buf); 2819 if (err) 2820 return err; 2821 } 2822 2823 return 0; 2824 } 2825 2826 int bpf_object__pin(struct bpf_object *obj, const char *path) 2827 { 2828 int err; 2829 2830 err = bpf_object__pin_maps(obj, path); 2831 if (err) 2832 return err; 2833 2834 err = bpf_object__pin_programs(obj, path); 2835 if (err) { 2836 bpf_object__unpin_maps(obj, path); 2837 return err; 2838 } 2839 2840 return 0; 2841 } 2842 2843 void bpf_object__close(struct bpf_object *obj) 2844 { 2845 size_t i; 2846 2847 if (!obj) 2848 return; 2849 2850 if (obj->clear_priv) 2851 obj->clear_priv(obj, obj->priv); 2852 2853 bpf_object__elf_finish(obj); 2854 bpf_object__unload(obj); 2855 btf__free(obj->btf); 2856 btf_ext__free(obj->btf_ext); 2857 2858 for (i = 0; i < obj->nr_maps; i++) { 2859 zfree(&obj->maps[i].name); 2860 if (obj->maps[i].clear_priv) 2861 obj->maps[i].clear_priv(&obj->maps[i], 2862 obj->maps[i].priv); 2863 obj->maps[i].priv = NULL; 2864 obj->maps[i].clear_priv = NULL; 2865 } 2866 2867 zfree(&obj->sections.rodata); 2868 zfree(&obj->sections.data); 2869 zfree(&obj->maps); 2870 obj->nr_maps = 0; 2871 2872 if (obj->programs && obj->nr_programs) { 2873 for (i = 0; i < obj->nr_programs; i++) 2874 bpf_program__exit(&obj->programs[i]); 2875 } 2876 zfree(&obj->programs); 2877 2878 list_del(&obj->list); 2879 free(obj); 2880 } 2881 2882 struct bpf_object * 2883 bpf_object__next(struct bpf_object *prev) 2884 { 2885 struct bpf_object *next; 2886 2887 if (!prev) 2888 next = list_first_entry(&bpf_objects_list, 2889 struct bpf_object, 2890 list); 2891 else 2892 next = list_next_entry(prev, list); 2893 2894 /* Empty list is noticed here so don't need checking on entry. */ 2895 if (&next->list == &bpf_objects_list) 2896 return NULL; 2897 2898 return next; 2899 } 2900 2901 const char *bpf_object__name(struct bpf_object *obj) 2902 { 2903 return obj ? obj->path : ERR_PTR(-EINVAL); 2904 } 2905 2906 unsigned int bpf_object__kversion(struct bpf_object *obj) 2907 { 2908 return obj ? obj->kern_version : 0; 2909 } 2910 2911 struct btf *bpf_object__btf(struct bpf_object *obj) 2912 { 2913 return obj ? obj->btf : NULL; 2914 } 2915 2916 int bpf_object__btf_fd(const struct bpf_object *obj) 2917 { 2918 return obj->btf ? btf__fd(obj->btf) : -1; 2919 } 2920 2921 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 2922 bpf_object_clear_priv_t clear_priv) 2923 { 2924 if (obj->priv && obj->clear_priv) 2925 obj->clear_priv(obj, obj->priv); 2926 2927 obj->priv = priv; 2928 obj->clear_priv = clear_priv; 2929 return 0; 2930 } 2931 2932 void *bpf_object__priv(struct bpf_object *obj) 2933 { 2934 return obj ? obj->priv : ERR_PTR(-EINVAL); 2935 } 2936 2937 static struct bpf_program * 2938 __bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward) 2939 { 2940 size_t nr_programs = obj->nr_programs; 2941 ssize_t idx; 2942 2943 if (!nr_programs) 2944 return NULL; 2945 2946 if (!p) 2947 /* Iter from the beginning */ 2948 return forward ? &obj->programs[0] : 2949 &obj->programs[nr_programs - 1]; 2950 2951 if (p->obj != obj) { 2952 pr_warning("error: program handler doesn't match object\n"); 2953 return NULL; 2954 } 2955 2956 idx = (p - obj->programs) + (forward ? 1 : -1); 2957 if (idx >= obj->nr_programs || idx < 0) 2958 return NULL; 2959 return &obj->programs[idx]; 2960 } 2961 2962 struct bpf_program * 2963 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 2964 { 2965 struct bpf_program *prog = prev; 2966 2967 do { 2968 prog = __bpf_program__iter(prog, obj, true); 2969 } while (prog && bpf_program__is_function_storage(prog, obj)); 2970 2971 return prog; 2972 } 2973 2974 struct bpf_program * 2975 bpf_program__prev(struct bpf_program *next, struct bpf_object *obj) 2976 { 2977 struct bpf_program *prog = next; 2978 2979 do { 2980 prog = __bpf_program__iter(prog, obj, false); 2981 } while (prog && bpf_program__is_function_storage(prog, obj)); 2982 2983 return prog; 2984 } 2985 2986 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 2987 bpf_program_clear_priv_t clear_priv) 2988 { 2989 if (prog->priv && prog->clear_priv) 2990 prog->clear_priv(prog, prog->priv); 2991 2992 prog->priv = priv; 2993 prog->clear_priv = clear_priv; 2994 return 0; 2995 } 2996 2997 void *bpf_program__priv(struct bpf_program *prog) 2998 { 2999 return prog ? prog->priv : ERR_PTR(-EINVAL); 3000 } 3001 3002 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 3003 { 3004 prog->prog_ifindex = ifindex; 3005 } 3006 3007 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 3008 { 3009 const char *title; 3010 3011 title = prog->section_name; 3012 if (needs_copy) { 3013 title = strdup(title); 3014 if (!title) { 3015 pr_warning("failed to strdup program title\n"); 3016 return ERR_PTR(-ENOMEM); 3017 } 3018 } 3019 3020 return title; 3021 } 3022 3023 int bpf_program__fd(struct bpf_program *prog) 3024 { 3025 return bpf_program__nth_fd(prog, 0); 3026 } 3027 3028 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 3029 bpf_program_prep_t prep) 3030 { 3031 int *instances_fds; 3032 3033 if (nr_instances <= 0 || !prep) 3034 return -EINVAL; 3035 3036 if (prog->instances.nr > 0 || prog->instances.fds) { 3037 pr_warning("Can't set pre-processor after loading\n"); 3038 return -EINVAL; 3039 } 3040 3041 instances_fds = malloc(sizeof(int) * nr_instances); 3042 if (!instances_fds) { 3043 pr_warning("alloc memory failed for fds\n"); 3044 return -ENOMEM; 3045 } 3046 3047 /* fill all fd with -1 */ 3048 memset(instances_fds, -1, sizeof(int) * nr_instances); 3049 3050 prog->instances.nr = nr_instances; 3051 prog->instances.fds = instances_fds; 3052 prog->preprocessor = prep; 3053 return 0; 3054 } 3055 3056 int bpf_program__nth_fd(struct bpf_program *prog, int n) 3057 { 3058 int fd; 3059 3060 if (!prog) 3061 return -EINVAL; 3062 3063 if (n >= prog->instances.nr || n < 0) { 3064 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 3065 n, prog->section_name, prog->instances.nr); 3066 return -EINVAL; 3067 } 3068 3069 fd = prog->instances.fds[n]; 3070 if (fd < 0) { 3071 pr_warning("%dth instance of program '%s' is invalid\n", 3072 n, prog->section_name); 3073 return -ENOENT; 3074 } 3075 3076 return fd; 3077 } 3078 3079 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 3080 { 3081 prog->type = type; 3082 } 3083 3084 static bool bpf_program__is_type(struct bpf_program *prog, 3085 enum bpf_prog_type type) 3086 { 3087 return prog ? (prog->type == type) : false; 3088 } 3089 3090 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 3091 int bpf_program__set_##NAME(struct bpf_program *prog) \ 3092 { \ 3093 if (!prog) \ 3094 return -EINVAL; \ 3095 bpf_program__set_type(prog, TYPE); \ 3096 return 0; \ 3097 } \ 3098 \ 3099 bool bpf_program__is_##NAME(struct bpf_program *prog) \ 3100 { \ 3101 return bpf_program__is_type(prog, TYPE); \ 3102 } \ 3103 3104 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 3105 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 3106 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 3107 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 3108 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 3109 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 3110 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 3111 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 3112 3113 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 3114 enum bpf_attach_type type) 3115 { 3116 prog->expected_attach_type = type; 3117 } 3118 3119 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ 3120 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } 3121 3122 /* Programs that can NOT be attached. */ 3123 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) 3124 3125 /* Programs that can be attached. */ 3126 #define BPF_APROG_SEC(string, ptype, atype) \ 3127 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) 3128 3129 /* Programs that must specify expected attach type at load time. */ 3130 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 3131 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) 3132 3133 /* Programs that can be attached but attach type can't be identified by section 3134 * name. Kept for backward compatibility. 3135 */ 3136 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 3137 3138 static const struct { 3139 const char *sec; 3140 size_t len; 3141 enum bpf_prog_type prog_type; 3142 enum bpf_attach_type expected_attach_type; 3143 int is_attachable; 3144 enum bpf_attach_type attach_type; 3145 } section_names[] = { 3146 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 3147 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 3148 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 3149 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 3150 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 3151 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 3152 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), 3153 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 3154 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 3155 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 3156 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 3157 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 3158 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 3159 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 3160 BPF_CGROUP_INET_INGRESS), 3161 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 3162 BPF_CGROUP_INET_EGRESS), 3163 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 3164 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 3165 BPF_CGROUP_INET_SOCK_CREATE), 3166 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 3167 BPF_CGROUP_INET4_POST_BIND), 3168 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 3169 BPF_CGROUP_INET6_POST_BIND), 3170 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 3171 BPF_CGROUP_DEVICE), 3172 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 3173 BPF_CGROUP_SOCK_OPS), 3174 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 3175 BPF_SK_SKB_STREAM_PARSER), 3176 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 3177 BPF_SK_SKB_STREAM_VERDICT), 3178 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 3179 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 3180 BPF_SK_MSG_VERDICT), 3181 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 3182 BPF_LIRC_MODE2), 3183 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 3184 BPF_FLOW_DISSECTOR), 3185 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3186 BPF_CGROUP_INET4_BIND), 3187 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3188 BPF_CGROUP_INET6_BIND), 3189 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3190 BPF_CGROUP_INET4_CONNECT), 3191 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3192 BPF_CGROUP_INET6_CONNECT), 3193 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3194 BPF_CGROUP_UDP4_SENDMSG), 3195 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3196 BPF_CGROUP_UDP6_SENDMSG), 3197 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 3198 BPF_CGROUP_SYSCTL), 3199 }; 3200 3201 #undef BPF_PROG_SEC_IMPL 3202 #undef BPF_PROG_SEC 3203 #undef BPF_APROG_SEC 3204 #undef BPF_EAPROG_SEC 3205 #undef BPF_APROG_COMPAT 3206 3207 #define MAX_TYPE_NAME_SIZE 32 3208 3209 static char *libbpf_get_type_names(bool attach_type) 3210 { 3211 int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; 3212 char *buf; 3213 3214 buf = malloc(len); 3215 if (!buf) 3216 return NULL; 3217 3218 buf[0] = '\0'; 3219 /* Forge string buf with all available names */ 3220 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3221 if (attach_type && !section_names[i].is_attachable) 3222 continue; 3223 3224 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { 3225 free(buf); 3226 return NULL; 3227 } 3228 strcat(buf, " "); 3229 strcat(buf, section_names[i].sec); 3230 } 3231 3232 return buf; 3233 } 3234 3235 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 3236 enum bpf_attach_type *expected_attach_type) 3237 { 3238 char *type_names; 3239 int i; 3240 3241 if (!name) 3242 return -EINVAL; 3243 3244 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3245 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3246 continue; 3247 *prog_type = section_names[i].prog_type; 3248 *expected_attach_type = section_names[i].expected_attach_type; 3249 return 0; 3250 } 3251 pr_warning("failed to guess program type based on ELF section name '%s'\n", name); 3252 type_names = libbpf_get_type_names(false); 3253 if (type_names != NULL) { 3254 pr_info("supported section(type) names are:%s\n", type_names); 3255 free(type_names); 3256 } 3257 3258 return -EINVAL; 3259 } 3260 3261 int libbpf_attach_type_by_name(const char *name, 3262 enum bpf_attach_type *attach_type) 3263 { 3264 char *type_names; 3265 int i; 3266 3267 if (!name) 3268 return -EINVAL; 3269 3270 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3271 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3272 continue; 3273 if (!section_names[i].is_attachable) 3274 return -EINVAL; 3275 *attach_type = section_names[i].attach_type; 3276 return 0; 3277 } 3278 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); 3279 type_names = libbpf_get_type_names(true); 3280 if (type_names != NULL) { 3281 pr_info("attachable section(type) names are:%s\n", type_names); 3282 free(type_names); 3283 } 3284 3285 return -EINVAL; 3286 } 3287 3288 static int 3289 bpf_program__identify_section(struct bpf_program *prog, 3290 enum bpf_prog_type *prog_type, 3291 enum bpf_attach_type *expected_attach_type) 3292 { 3293 return libbpf_prog_type_by_name(prog->section_name, prog_type, 3294 expected_attach_type); 3295 } 3296 3297 int bpf_map__fd(struct bpf_map *map) 3298 { 3299 return map ? map->fd : -EINVAL; 3300 } 3301 3302 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 3303 { 3304 return map ? &map->def : ERR_PTR(-EINVAL); 3305 } 3306 3307 const char *bpf_map__name(struct bpf_map *map) 3308 { 3309 return map ? map->name : NULL; 3310 } 3311 3312 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 3313 { 3314 return map ? map->btf_key_type_id : 0; 3315 } 3316 3317 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 3318 { 3319 return map ? map->btf_value_type_id : 0; 3320 } 3321 3322 int bpf_map__set_priv(struct bpf_map *map, void *priv, 3323 bpf_map_clear_priv_t clear_priv) 3324 { 3325 if (!map) 3326 return -EINVAL; 3327 3328 if (map->priv) { 3329 if (map->clear_priv) 3330 map->clear_priv(map, map->priv); 3331 } 3332 3333 map->priv = priv; 3334 map->clear_priv = clear_priv; 3335 return 0; 3336 } 3337 3338 void *bpf_map__priv(struct bpf_map *map) 3339 { 3340 return map ? map->priv : ERR_PTR(-EINVAL); 3341 } 3342 3343 bool bpf_map__is_offload_neutral(struct bpf_map *map) 3344 { 3345 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 3346 } 3347 3348 bool bpf_map__is_internal(struct bpf_map *map) 3349 { 3350 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 3351 } 3352 3353 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 3354 { 3355 map->map_ifindex = ifindex; 3356 } 3357 3358 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 3359 { 3360 if (!bpf_map_type__is_map_in_map(map->def.type)) { 3361 pr_warning("error: unsupported map type\n"); 3362 return -EINVAL; 3363 } 3364 if (map->inner_map_fd != -1) { 3365 pr_warning("error: inner_map_fd already specified\n"); 3366 return -EINVAL; 3367 } 3368 map->inner_map_fd = fd; 3369 return 0; 3370 } 3371 3372 static struct bpf_map * 3373 __bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i) 3374 { 3375 ssize_t idx; 3376 struct bpf_map *s, *e; 3377 3378 if (!obj || !obj->maps) 3379 return NULL; 3380 3381 s = obj->maps; 3382 e = obj->maps + obj->nr_maps; 3383 3384 if ((m < s) || (m >= e)) { 3385 pr_warning("error in %s: map handler doesn't belong to object\n", 3386 __func__); 3387 return NULL; 3388 } 3389 3390 idx = (m - obj->maps) + i; 3391 if (idx >= obj->nr_maps || idx < 0) 3392 return NULL; 3393 return &obj->maps[idx]; 3394 } 3395 3396 struct bpf_map * 3397 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 3398 { 3399 if (prev == NULL) 3400 return obj->maps; 3401 3402 return __bpf_map__iter(prev, obj, 1); 3403 } 3404 3405 struct bpf_map * 3406 bpf_map__prev(struct bpf_map *next, struct bpf_object *obj) 3407 { 3408 if (next == NULL) { 3409 if (!obj->nr_maps) 3410 return NULL; 3411 return obj->maps + obj->nr_maps - 1; 3412 } 3413 3414 return __bpf_map__iter(next, obj, -1); 3415 } 3416 3417 struct bpf_map * 3418 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 3419 { 3420 struct bpf_map *pos; 3421 3422 bpf_object__for_each_map(pos, obj) { 3423 if (pos->name && !strcmp(pos->name, name)) 3424 return pos; 3425 } 3426 return NULL; 3427 } 3428 3429 int 3430 bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name) 3431 { 3432 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 3433 } 3434 3435 struct bpf_map * 3436 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 3437 { 3438 int i; 3439 3440 for (i = 0; i < obj->nr_maps; i++) { 3441 if (obj->maps[i].offset == offset) 3442 return &obj->maps[i]; 3443 } 3444 return ERR_PTR(-ENOENT); 3445 } 3446 3447 long libbpf_get_error(const void *ptr) 3448 { 3449 return PTR_ERR_OR_ZERO(ptr); 3450 } 3451 3452 int bpf_prog_load(const char *file, enum bpf_prog_type type, 3453 struct bpf_object **pobj, int *prog_fd) 3454 { 3455 struct bpf_prog_load_attr attr; 3456 3457 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 3458 attr.file = file; 3459 attr.prog_type = type; 3460 attr.expected_attach_type = 0; 3461 3462 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 3463 } 3464 3465 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 3466 struct bpf_object **pobj, int *prog_fd) 3467 { 3468 struct bpf_object_open_attr open_attr = { 3469 .file = attr->file, 3470 .prog_type = attr->prog_type, 3471 }; 3472 struct bpf_program *prog, *first_prog = NULL; 3473 enum bpf_attach_type expected_attach_type; 3474 enum bpf_prog_type prog_type; 3475 struct bpf_object *obj; 3476 struct bpf_map *map; 3477 int err; 3478 3479 if (!attr) 3480 return -EINVAL; 3481 if (!attr->file) 3482 return -EINVAL; 3483 3484 obj = bpf_object__open_xattr(&open_attr); 3485 if (IS_ERR_OR_NULL(obj)) 3486 return -ENOENT; 3487 3488 bpf_object__for_each_program(prog, obj) { 3489 /* 3490 * If type is not specified, try to guess it based on 3491 * section name. 3492 */ 3493 prog_type = attr->prog_type; 3494 prog->prog_ifindex = attr->ifindex; 3495 expected_attach_type = attr->expected_attach_type; 3496 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 3497 err = bpf_program__identify_section(prog, &prog_type, 3498 &expected_attach_type); 3499 if (err < 0) { 3500 bpf_object__close(obj); 3501 return -EINVAL; 3502 } 3503 } 3504 3505 bpf_program__set_type(prog, prog_type); 3506 bpf_program__set_expected_attach_type(prog, 3507 expected_attach_type); 3508 3509 prog->log_level = attr->log_level; 3510 prog->prog_flags = attr->prog_flags; 3511 if (!first_prog) 3512 first_prog = prog; 3513 } 3514 3515 bpf_object__for_each_map(map, obj) { 3516 if (!bpf_map__is_offload_neutral(map)) 3517 map->map_ifindex = attr->ifindex; 3518 } 3519 3520 if (!first_prog) { 3521 pr_warning("object file doesn't contain bpf program\n"); 3522 bpf_object__close(obj); 3523 return -ENOENT; 3524 } 3525 3526 err = bpf_object__load(obj); 3527 if (err) { 3528 bpf_object__close(obj); 3529 return -EINVAL; 3530 } 3531 3532 *pobj = obj; 3533 *prog_fd = bpf_program__fd(first_prog); 3534 return 0; 3535 } 3536 3537 enum bpf_perf_event_ret 3538 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 3539 void **copy_mem, size_t *copy_size, 3540 bpf_perf_event_print_t fn, void *private_data) 3541 { 3542 struct perf_event_mmap_page *header = mmap_mem; 3543 __u64 data_head = ring_buffer_read_head(header); 3544 __u64 data_tail = header->data_tail; 3545 void *base = ((__u8 *)header) + page_size; 3546 int ret = LIBBPF_PERF_EVENT_CONT; 3547 struct perf_event_header *ehdr; 3548 size_t ehdr_size; 3549 3550 while (data_head != data_tail) { 3551 ehdr = base + (data_tail & (mmap_size - 1)); 3552 ehdr_size = ehdr->size; 3553 3554 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 3555 void *copy_start = ehdr; 3556 size_t len_first = base + mmap_size - copy_start; 3557 size_t len_secnd = ehdr_size - len_first; 3558 3559 if (*copy_size < ehdr_size) { 3560 free(*copy_mem); 3561 *copy_mem = malloc(ehdr_size); 3562 if (!*copy_mem) { 3563 *copy_size = 0; 3564 ret = LIBBPF_PERF_EVENT_ERROR; 3565 break; 3566 } 3567 *copy_size = ehdr_size; 3568 } 3569 3570 memcpy(*copy_mem, copy_start, len_first); 3571 memcpy(*copy_mem + len_first, base, len_secnd); 3572 ehdr = *copy_mem; 3573 } 3574 3575 ret = fn(ehdr, private_data); 3576 data_tail += ehdr_size; 3577 if (ret != LIBBPF_PERF_EVENT_CONT) 3578 break; 3579 } 3580 3581 ring_buffer_write_tail(header, data_tail); 3582 return ret; 3583 } 3584 3585 struct bpf_prog_info_array_desc { 3586 int array_offset; /* e.g. offset of jited_prog_insns */ 3587 int count_offset; /* e.g. offset of jited_prog_len */ 3588 int size_offset; /* > 0: offset of rec size, 3589 * < 0: fix size of -size_offset 3590 */ 3591 }; 3592 3593 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 3594 [BPF_PROG_INFO_JITED_INSNS] = { 3595 offsetof(struct bpf_prog_info, jited_prog_insns), 3596 offsetof(struct bpf_prog_info, jited_prog_len), 3597 -1, 3598 }, 3599 [BPF_PROG_INFO_XLATED_INSNS] = { 3600 offsetof(struct bpf_prog_info, xlated_prog_insns), 3601 offsetof(struct bpf_prog_info, xlated_prog_len), 3602 -1, 3603 }, 3604 [BPF_PROG_INFO_MAP_IDS] = { 3605 offsetof(struct bpf_prog_info, map_ids), 3606 offsetof(struct bpf_prog_info, nr_map_ids), 3607 -(int)sizeof(__u32), 3608 }, 3609 [BPF_PROG_INFO_JITED_KSYMS] = { 3610 offsetof(struct bpf_prog_info, jited_ksyms), 3611 offsetof(struct bpf_prog_info, nr_jited_ksyms), 3612 -(int)sizeof(__u64), 3613 }, 3614 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 3615 offsetof(struct bpf_prog_info, jited_func_lens), 3616 offsetof(struct bpf_prog_info, nr_jited_func_lens), 3617 -(int)sizeof(__u32), 3618 }, 3619 [BPF_PROG_INFO_FUNC_INFO] = { 3620 offsetof(struct bpf_prog_info, func_info), 3621 offsetof(struct bpf_prog_info, nr_func_info), 3622 offsetof(struct bpf_prog_info, func_info_rec_size), 3623 }, 3624 [BPF_PROG_INFO_LINE_INFO] = { 3625 offsetof(struct bpf_prog_info, line_info), 3626 offsetof(struct bpf_prog_info, nr_line_info), 3627 offsetof(struct bpf_prog_info, line_info_rec_size), 3628 }, 3629 [BPF_PROG_INFO_JITED_LINE_INFO] = { 3630 offsetof(struct bpf_prog_info, jited_line_info), 3631 offsetof(struct bpf_prog_info, nr_jited_line_info), 3632 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 3633 }, 3634 [BPF_PROG_INFO_PROG_TAGS] = { 3635 offsetof(struct bpf_prog_info, prog_tags), 3636 offsetof(struct bpf_prog_info, nr_prog_tags), 3637 -(int)sizeof(__u8) * BPF_TAG_SIZE, 3638 }, 3639 3640 }; 3641 3642 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) 3643 { 3644 __u32 *array = (__u32 *)info; 3645 3646 if (offset >= 0) 3647 return array[offset / sizeof(__u32)]; 3648 return -(int)offset; 3649 } 3650 3651 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) 3652 { 3653 __u64 *array = (__u64 *)info; 3654 3655 if (offset >= 0) 3656 return array[offset / sizeof(__u64)]; 3657 return -(int)offset; 3658 } 3659 3660 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 3661 __u32 val) 3662 { 3663 __u32 *array = (__u32 *)info; 3664 3665 if (offset >= 0) 3666 array[offset / sizeof(__u32)] = val; 3667 } 3668 3669 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 3670 __u64 val) 3671 { 3672 __u64 *array = (__u64 *)info; 3673 3674 if (offset >= 0) 3675 array[offset / sizeof(__u64)] = val; 3676 } 3677 3678 struct bpf_prog_info_linear * 3679 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 3680 { 3681 struct bpf_prog_info_linear *info_linear; 3682 struct bpf_prog_info info = {}; 3683 __u32 info_len = sizeof(info); 3684 __u32 data_len = 0; 3685 int i, err; 3686 void *ptr; 3687 3688 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 3689 return ERR_PTR(-EINVAL); 3690 3691 /* step 1: get array dimensions */ 3692 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 3693 if (err) { 3694 pr_debug("can't get prog info: %s", strerror(errno)); 3695 return ERR_PTR(-EFAULT); 3696 } 3697 3698 /* step 2: calculate total size of all arrays */ 3699 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3700 bool include_array = (arrays & (1UL << i)) > 0; 3701 struct bpf_prog_info_array_desc *desc; 3702 __u32 count, size; 3703 3704 desc = bpf_prog_info_array_desc + i; 3705 3706 /* kernel is too old to support this field */ 3707 if (info_len < desc->array_offset + sizeof(__u32) || 3708 info_len < desc->count_offset + sizeof(__u32) || 3709 (desc->size_offset > 0 && info_len < desc->size_offset)) 3710 include_array = false; 3711 3712 if (!include_array) { 3713 arrays &= ~(1UL << i); /* clear the bit */ 3714 continue; 3715 } 3716 3717 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3718 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3719 3720 data_len += count * size; 3721 } 3722 3723 /* step 3: allocate continuous memory */ 3724 data_len = roundup(data_len, sizeof(__u64)); 3725 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 3726 if (!info_linear) 3727 return ERR_PTR(-ENOMEM); 3728 3729 /* step 4: fill data to info_linear->info */ 3730 info_linear->arrays = arrays; 3731 memset(&info_linear->info, 0, sizeof(info)); 3732 ptr = info_linear->data; 3733 3734 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3735 struct bpf_prog_info_array_desc *desc; 3736 __u32 count, size; 3737 3738 if ((arrays & (1UL << i)) == 0) 3739 continue; 3740 3741 desc = bpf_prog_info_array_desc + i; 3742 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3743 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3744 bpf_prog_info_set_offset_u32(&info_linear->info, 3745 desc->count_offset, count); 3746 bpf_prog_info_set_offset_u32(&info_linear->info, 3747 desc->size_offset, size); 3748 bpf_prog_info_set_offset_u64(&info_linear->info, 3749 desc->array_offset, 3750 ptr_to_u64(ptr)); 3751 ptr += count * size; 3752 } 3753 3754 /* step 5: call syscall again to get required arrays */ 3755 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 3756 if (err) { 3757 pr_debug("can't get prog info: %s", strerror(errno)); 3758 free(info_linear); 3759 return ERR_PTR(-EFAULT); 3760 } 3761 3762 /* step 6: verify the data */ 3763 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3764 struct bpf_prog_info_array_desc *desc; 3765 __u32 v1, v2; 3766 3767 if ((arrays & (1UL << i)) == 0) 3768 continue; 3769 3770 desc = bpf_prog_info_array_desc + i; 3771 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3772 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3773 desc->count_offset); 3774 if (v1 != v2) 3775 pr_warning("%s: mismatch in element count\n", __func__); 3776 3777 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3778 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3779 desc->size_offset); 3780 if (v1 != v2) 3781 pr_warning("%s: mismatch in rec size\n", __func__); 3782 } 3783 3784 /* step 7: update info_len and data_len */ 3785 info_linear->info_len = sizeof(struct bpf_prog_info); 3786 info_linear->data_len = data_len; 3787 3788 return info_linear; 3789 } 3790 3791 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 3792 { 3793 int i; 3794 3795 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3796 struct bpf_prog_info_array_desc *desc; 3797 __u64 addr, offs; 3798 3799 if ((info_linear->arrays & (1UL << i)) == 0) 3800 continue; 3801 3802 desc = bpf_prog_info_array_desc + i; 3803 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 3804 desc->array_offset); 3805 offs = addr - ptr_to_u64(info_linear->data); 3806 bpf_prog_info_set_offset_u64(&info_linear->info, 3807 desc->array_offset, offs); 3808 } 3809 } 3810 3811 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 3812 { 3813 int i; 3814 3815 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3816 struct bpf_prog_info_array_desc *desc; 3817 __u64 addr, offs; 3818 3819 if ((info_linear->arrays & (1UL << i)) == 0) 3820 continue; 3821 3822 desc = bpf_prog_info_array_desc + i; 3823 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 3824 desc->array_offset); 3825 addr = offs + ptr_to_u64(info_linear->data); 3826 bpf_prog_info_set_offset_u64(&info_linear->info, 3827 desc->array_offset, addr); 3828 } 3829 } 3830