1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <string.h> 22 #include <unistd.h> 23 #include <fcntl.h> 24 #include <errno.h> 25 #include <asm/unistd.h> 26 #include <linux/err.h> 27 #include <linux/kernel.h> 28 #include <linux/bpf.h> 29 #include <linux/btf.h> 30 #include <linux/filter.h> 31 #include <linux/list.h> 32 #include <linux/limits.h> 33 #include <linux/perf_event.h> 34 #include <linux/ring_buffer.h> 35 #include <sys/stat.h> 36 #include <sys/types.h> 37 #include <sys/vfs.h> 38 #include <tools/libc_compat.h> 39 #include <libelf.h> 40 #include <gelf.h> 41 42 #include "libbpf.h" 43 #include "bpf.h" 44 #include "btf.h" 45 #include "str_error.h" 46 #include "libbpf_util.h" 47 48 #ifndef EM_BPF 49 #define EM_BPF 247 50 #endif 51 52 #ifndef BPF_FS_MAGIC 53 #define BPF_FS_MAGIC 0xcafe4a11 54 #endif 55 56 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 57 * compilation if user enables corresponding warning. Disable it explicitly. 58 */ 59 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 60 61 #define __printf(a, b) __attribute__((format(printf, a, b))) 62 63 static int __base_pr(enum libbpf_print_level level, const char *format, 64 va_list args) 65 { 66 if (level == LIBBPF_DEBUG) 67 return 0; 68 69 return vfprintf(stderr, format, args); 70 } 71 72 static libbpf_print_fn_t __libbpf_pr = __base_pr; 73 74 void libbpf_set_print(libbpf_print_fn_t fn) 75 { 76 __libbpf_pr = fn; 77 } 78 79 __printf(2, 3) 80 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 81 { 82 va_list args; 83 84 if (!__libbpf_pr) 85 return; 86 87 va_start(args, format); 88 __libbpf_pr(level, format, args); 89 va_end(args); 90 } 91 92 #define STRERR_BUFSIZE 128 93 94 #define CHECK_ERR(action, err, out) do { \ 95 err = action; \ 96 if (err) \ 97 goto out; \ 98 } while(0) 99 100 101 /* Copied from tools/perf/util/util.h */ 102 #ifndef zfree 103 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 104 #endif 105 106 #ifndef zclose 107 # define zclose(fd) ({ \ 108 int ___err = 0; \ 109 if ((fd) >= 0) \ 110 ___err = close((fd)); \ 111 fd = -1; \ 112 ___err; }) 113 #endif 114 115 #ifdef HAVE_LIBELF_MMAP_SUPPORT 116 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 117 #else 118 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 119 #endif 120 121 static inline __u64 ptr_to_u64(const void *ptr) 122 { 123 return (__u64) (unsigned long) ptr; 124 } 125 126 struct bpf_capabilities { 127 /* v4.14: kernel support for program & map names. */ 128 __u32 name:1; 129 /* v5.2: kernel support for global data sections. */ 130 __u32 global_data:1; 131 }; 132 133 /* 134 * bpf_prog should be a better name but it has been used in 135 * linux/filter.h. 136 */ 137 struct bpf_program { 138 /* Index in elf obj file, for relocation use. */ 139 int idx; 140 char *name; 141 int prog_ifindex; 142 char *section_name; 143 /* section_name with / replaced by _; makes recursive pinning 144 * in bpf_object__pin_programs easier 145 */ 146 char *pin_name; 147 struct bpf_insn *insns; 148 size_t insns_cnt, main_prog_cnt; 149 enum bpf_prog_type type; 150 151 struct reloc_desc { 152 enum { 153 RELO_LD64, 154 RELO_CALL, 155 RELO_DATA, 156 } type; 157 int insn_idx; 158 union { 159 int map_idx; 160 int text_off; 161 }; 162 } *reloc_desc; 163 int nr_reloc; 164 int log_level; 165 166 struct { 167 int nr; 168 int *fds; 169 } instances; 170 bpf_program_prep_t preprocessor; 171 172 struct bpf_object *obj; 173 void *priv; 174 bpf_program_clear_priv_t clear_priv; 175 176 enum bpf_attach_type expected_attach_type; 177 int btf_fd; 178 void *func_info; 179 __u32 func_info_rec_size; 180 __u32 func_info_cnt; 181 182 struct bpf_capabilities *caps; 183 184 void *line_info; 185 __u32 line_info_rec_size; 186 __u32 line_info_cnt; 187 }; 188 189 enum libbpf_map_type { 190 LIBBPF_MAP_UNSPEC, 191 LIBBPF_MAP_DATA, 192 LIBBPF_MAP_BSS, 193 LIBBPF_MAP_RODATA, 194 }; 195 196 static const char * const libbpf_type_to_btf_name[] = { 197 [LIBBPF_MAP_DATA] = ".data", 198 [LIBBPF_MAP_BSS] = ".bss", 199 [LIBBPF_MAP_RODATA] = ".rodata", 200 }; 201 202 struct bpf_map { 203 int fd; 204 char *name; 205 size_t offset; 206 int map_ifindex; 207 int inner_map_fd; 208 struct bpf_map_def def; 209 __u32 btf_key_type_id; 210 __u32 btf_value_type_id; 211 void *priv; 212 bpf_map_clear_priv_t clear_priv; 213 enum libbpf_map_type libbpf_type; 214 }; 215 216 struct bpf_secdata { 217 void *rodata; 218 void *data; 219 }; 220 221 static LIST_HEAD(bpf_objects_list); 222 223 struct bpf_object { 224 char name[BPF_OBJ_NAME_LEN]; 225 char license[64]; 226 __u32 kern_version; 227 228 struct bpf_program *programs; 229 size_t nr_programs; 230 struct bpf_map *maps; 231 size_t nr_maps; 232 struct bpf_secdata sections; 233 234 bool loaded; 235 bool has_pseudo_calls; 236 237 /* 238 * Information when doing elf related work. Only valid if fd 239 * is valid. 240 */ 241 struct { 242 int fd; 243 void *obj_buf; 244 size_t obj_buf_sz; 245 Elf *elf; 246 GElf_Ehdr ehdr; 247 Elf_Data *symbols; 248 Elf_Data *data; 249 Elf_Data *rodata; 250 Elf_Data *bss; 251 size_t strtabidx; 252 struct { 253 GElf_Shdr shdr; 254 Elf_Data *data; 255 } *reloc; 256 int nr_reloc; 257 int maps_shndx; 258 int text_shndx; 259 int data_shndx; 260 int rodata_shndx; 261 int bss_shndx; 262 } efile; 263 /* 264 * All loaded bpf_object is linked in a list, which is 265 * hidden to caller. bpf_objects__<func> handlers deal with 266 * all objects. 267 */ 268 struct list_head list; 269 270 struct btf *btf; 271 struct btf_ext *btf_ext; 272 273 void *priv; 274 bpf_object_clear_priv_t clear_priv; 275 276 struct bpf_capabilities caps; 277 278 char path[]; 279 }; 280 #define obj_elf_valid(o) ((o)->efile.elf) 281 282 void bpf_program__unload(struct bpf_program *prog) 283 { 284 int i; 285 286 if (!prog) 287 return; 288 289 /* 290 * If the object is opened but the program was never loaded, 291 * it is possible that prog->instances.nr == -1. 292 */ 293 if (prog->instances.nr > 0) { 294 for (i = 0; i < prog->instances.nr; i++) 295 zclose(prog->instances.fds[i]); 296 } else if (prog->instances.nr != -1) { 297 pr_warning("Internal error: instances.nr is %d\n", 298 prog->instances.nr); 299 } 300 301 prog->instances.nr = -1; 302 zfree(&prog->instances.fds); 303 304 zclose(prog->btf_fd); 305 zfree(&prog->func_info); 306 zfree(&prog->line_info); 307 } 308 309 static void bpf_program__exit(struct bpf_program *prog) 310 { 311 if (!prog) 312 return; 313 314 if (prog->clear_priv) 315 prog->clear_priv(prog, prog->priv); 316 317 prog->priv = NULL; 318 prog->clear_priv = NULL; 319 320 bpf_program__unload(prog); 321 zfree(&prog->name); 322 zfree(&prog->section_name); 323 zfree(&prog->pin_name); 324 zfree(&prog->insns); 325 zfree(&prog->reloc_desc); 326 327 prog->nr_reloc = 0; 328 prog->insns_cnt = 0; 329 prog->idx = -1; 330 } 331 332 static char *__bpf_program__pin_name(struct bpf_program *prog) 333 { 334 char *name, *p; 335 336 name = p = strdup(prog->section_name); 337 while ((p = strchr(p, '/'))) 338 *p = '_'; 339 340 return name; 341 } 342 343 static int 344 bpf_program__init(void *data, size_t size, char *section_name, int idx, 345 struct bpf_program *prog) 346 { 347 if (size < sizeof(struct bpf_insn)) { 348 pr_warning("corrupted section '%s'\n", section_name); 349 return -EINVAL; 350 } 351 352 memset(prog, 0, sizeof(*prog)); 353 354 prog->section_name = strdup(section_name); 355 if (!prog->section_name) { 356 pr_warning("failed to alloc name for prog under section(%d) %s\n", 357 idx, section_name); 358 goto errout; 359 } 360 361 prog->pin_name = __bpf_program__pin_name(prog); 362 if (!prog->pin_name) { 363 pr_warning("failed to alloc pin name for prog under section(%d) %s\n", 364 idx, section_name); 365 goto errout; 366 } 367 368 prog->insns = malloc(size); 369 if (!prog->insns) { 370 pr_warning("failed to alloc insns for prog under section %s\n", 371 section_name); 372 goto errout; 373 } 374 prog->insns_cnt = size / sizeof(struct bpf_insn); 375 memcpy(prog->insns, data, 376 prog->insns_cnt * sizeof(struct bpf_insn)); 377 prog->idx = idx; 378 prog->instances.fds = NULL; 379 prog->instances.nr = -1; 380 prog->type = BPF_PROG_TYPE_UNSPEC; 381 prog->btf_fd = -1; 382 383 return 0; 384 errout: 385 bpf_program__exit(prog); 386 return -ENOMEM; 387 } 388 389 static int 390 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 391 char *section_name, int idx) 392 { 393 struct bpf_program prog, *progs; 394 int nr_progs, err; 395 396 err = bpf_program__init(data, size, section_name, idx, &prog); 397 if (err) 398 return err; 399 400 prog.caps = &obj->caps; 401 progs = obj->programs; 402 nr_progs = obj->nr_programs; 403 404 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 405 if (!progs) { 406 /* 407 * In this case the original obj->programs 408 * is still valid, so don't need special treat for 409 * bpf_close_object(). 410 */ 411 pr_warning("failed to alloc a new program under section '%s'\n", 412 section_name); 413 bpf_program__exit(&prog); 414 return -ENOMEM; 415 } 416 417 pr_debug("found program %s\n", prog.section_name); 418 obj->programs = progs; 419 obj->nr_programs = nr_progs + 1; 420 prog.obj = obj; 421 progs[nr_progs] = prog; 422 return 0; 423 } 424 425 static int 426 bpf_object__init_prog_names(struct bpf_object *obj) 427 { 428 Elf_Data *symbols = obj->efile.symbols; 429 struct bpf_program *prog; 430 size_t pi, si; 431 432 for (pi = 0; pi < obj->nr_programs; pi++) { 433 const char *name = NULL; 434 435 prog = &obj->programs[pi]; 436 437 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 438 si++) { 439 GElf_Sym sym; 440 441 if (!gelf_getsym(symbols, si, &sym)) 442 continue; 443 if (sym.st_shndx != prog->idx) 444 continue; 445 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 446 continue; 447 448 name = elf_strptr(obj->efile.elf, 449 obj->efile.strtabidx, 450 sym.st_name); 451 if (!name) { 452 pr_warning("failed to get sym name string for prog %s\n", 453 prog->section_name); 454 return -LIBBPF_ERRNO__LIBELF; 455 } 456 } 457 458 if (!name && prog->idx == obj->efile.text_shndx) 459 name = ".text"; 460 461 if (!name) { 462 pr_warning("failed to find sym for prog %s\n", 463 prog->section_name); 464 return -EINVAL; 465 } 466 467 prog->name = strdup(name); 468 if (!prog->name) { 469 pr_warning("failed to allocate memory for prog sym %s\n", 470 name); 471 return -ENOMEM; 472 } 473 } 474 475 return 0; 476 } 477 478 static struct bpf_object *bpf_object__new(const char *path, 479 void *obj_buf, 480 size_t obj_buf_sz) 481 { 482 struct bpf_object *obj; 483 char *end; 484 485 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 486 if (!obj) { 487 pr_warning("alloc memory failed for %s\n", path); 488 return ERR_PTR(-ENOMEM); 489 } 490 491 strcpy(obj->path, path); 492 /* Using basename() GNU version which doesn't modify arg. */ 493 strncpy(obj->name, basename((void *)path), 494 sizeof(obj->name) - 1); 495 end = strchr(obj->name, '.'); 496 if (end) 497 *end = 0; 498 499 obj->efile.fd = -1; 500 /* 501 * Caller of this function should also calls 502 * bpf_object__elf_finish() after data collection to return 503 * obj_buf to user. If not, we should duplicate the buffer to 504 * avoid user freeing them before elf finish. 505 */ 506 obj->efile.obj_buf = obj_buf; 507 obj->efile.obj_buf_sz = obj_buf_sz; 508 obj->efile.maps_shndx = -1; 509 obj->efile.data_shndx = -1; 510 obj->efile.rodata_shndx = -1; 511 obj->efile.bss_shndx = -1; 512 513 obj->loaded = false; 514 515 INIT_LIST_HEAD(&obj->list); 516 list_add(&obj->list, &bpf_objects_list); 517 return obj; 518 } 519 520 static void bpf_object__elf_finish(struct bpf_object *obj) 521 { 522 if (!obj_elf_valid(obj)) 523 return; 524 525 if (obj->efile.elf) { 526 elf_end(obj->efile.elf); 527 obj->efile.elf = NULL; 528 } 529 obj->efile.symbols = NULL; 530 obj->efile.data = NULL; 531 obj->efile.rodata = NULL; 532 obj->efile.bss = NULL; 533 534 zfree(&obj->efile.reloc); 535 obj->efile.nr_reloc = 0; 536 zclose(obj->efile.fd); 537 obj->efile.obj_buf = NULL; 538 obj->efile.obj_buf_sz = 0; 539 } 540 541 static int bpf_object__elf_init(struct bpf_object *obj) 542 { 543 int err = 0; 544 GElf_Ehdr *ep; 545 546 if (obj_elf_valid(obj)) { 547 pr_warning("elf init: internal error\n"); 548 return -LIBBPF_ERRNO__LIBELF; 549 } 550 551 if (obj->efile.obj_buf_sz > 0) { 552 /* 553 * obj_buf should have been validated by 554 * bpf_object__open_buffer(). 555 */ 556 obj->efile.elf = elf_memory(obj->efile.obj_buf, 557 obj->efile.obj_buf_sz); 558 } else { 559 obj->efile.fd = open(obj->path, O_RDONLY); 560 if (obj->efile.fd < 0) { 561 char errmsg[STRERR_BUFSIZE]; 562 char *cp = libbpf_strerror_r(errno, errmsg, 563 sizeof(errmsg)); 564 565 pr_warning("failed to open %s: %s\n", obj->path, cp); 566 return -errno; 567 } 568 569 obj->efile.elf = elf_begin(obj->efile.fd, 570 LIBBPF_ELF_C_READ_MMAP, 571 NULL); 572 } 573 574 if (!obj->efile.elf) { 575 pr_warning("failed to open %s as ELF file\n", 576 obj->path); 577 err = -LIBBPF_ERRNO__LIBELF; 578 goto errout; 579 } 580 581 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 582 pr_warning("failed to get EHDR from %s\n", 583 obj->path); 584 err = -LIBBPF_ERRNO__FORMAT; 585 goto errout; 586 } 587 ep = &obj->efile.ehdr; 588 589 /* Old LLVM set e_machine to EM_NONE */ 590 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { 591 pr_warning("%s is not an eBPF object file\n", 592 obj->path); 593 err = -LIBBPF_ERRNO__FORMAT; 594 goto errout; 595 } 596 597 return 0; 598 errout: 599 bpf_object__elf_finish(obj); 600 return err; 601 } 602 603 static int 604 bpf_object__check_endianness(struct bpf_object *obj) 605 { 606 static unsigned int const endian = 1; 607 608 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 609 case ELFDATA2LSB: 610 /* We are big endian, BPF obj is little endian. */ 611 if (*(unsigned char const *)&endian != 1) 612 goto mismatch; 613 break; 614 615 case ELFDATA2MSB: 616 /* We are little endian, BPF obj is big endian. */ 617 if (*(unsigned char const *)&endian != 0) 618 goto mismatch; 619 break; 620 default: 621 return -LIBBPF_ERRNO__ENDIAN; 622 } 623 624 return 0; 625 626 mismatch: 627 pr_warning("Error: endianness mismatch.\n"); 628 return -LIBBPF_ERRNO__ENDIAN; 629 } 630 631 static int 632 bpf_object__init_license(struct bpf_object *obj, 633 void *data, size_t size) 634 { 635 memcpy(obj->license, data, 636 min(size, sizeof(obj->license) - 1)); 637 pr_debug("license of %s is %s\n", obj->path, obj->license); 638 return 0; 639 } 640 641 static int 642 bpf_object__init_kversion(struct bpf_object *obj, 643 void *data, size_t size) 644 { 645 __u32 kver; 646 647 if (size != sizeof(kver)) { 648 pr_warning("invalid kver section in %s\n", obj->path); 649 return -LIBBPF_ERRNO__FORMAT; 650 } 651 memcpy(&kver, data, sizeof(kver)); 652 obj->kern_version = kver; 653 pr_debug("kernel version of %s is %x\n", obj->path, 654 obj->kern_version); 655 return 0; 656 } 657 658 static int compare_bpf_map(const void *_a, const void *_b) 659 { 660 const struct bpf_map *a = _a; 661 const struct bpf_map *b = _b; 662 663 return a->offset - b->offset; 664 } 665 666 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 667 { 668 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 669 type == BPF_MAP_TYPE_HASH_OF_MAPS) 670 return true; 671 return false; 672 } 673 674 static int bpf_object_search_section_size(const struct bpf_object *obj, 675 const char *name, size_t *d_size) 676 { 677 const GElf_Ehdr *ep = &obj->efile.ehdr; 678 Elf *elf = obj->efile.elf; 679 Elf_Scn *scn = NULL; 680 int idx = 0; 681 682 while ((scn = elf_nextscn(elf, scn)) != NULL) { 683 const char *sec_name; 684 Elf_Data *data; 685 GElf_Shdr sh; 686 687 idx++; 688 if (gelf_getshdr(scn, &sh) != &sh) { 689 pr_warning("failed to get section(%d) header from %s\n", 690 idx, obj->path); 691 return -EIO; 692 } 693 694 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 695 if (!sec_name) { 696 pr_warning("failed to get section(%d) name from %s\n", 697 idx, obj->path); 698 return -EIO; 699 } 700 701 if (strcmp(name, sec_name)) 702 continue; 703 704 data = elf_getdata(scn, 0); 705 if (!data) { 706 pr_warning("failed to get section(%d) data from %s(%s)\n", 707 idx, name, obj->path); 708 return -EIO; 709 } 710 711 *d_size = data->d_size; 712 return 0; 713 } 714 715 return -ENOENT; 716 } 717 718 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 719 __u32 *size) 720 { 721 int ret = -ENOENT; 722 size_t d_size; 723 724 *size = 0; 725 if (!name) { 726 return -EINVAL; 727 } else if (!strcmp(name, ".data")) { 728 if (obj->efile.data) 729 *size = obj->efile.data->d_size; 730 } else if (!strcmp(name, ".bss")) { 731 if (obj->efile.bss) 732 *size = obj->efile.bss->d_size; 733 } else if (!strcmp(name, ".rodata")) { 734 if (obj->efile.rodata) 735 *size = obj->efile.rodata->d_size; 736 } else { 737 ret = bpf_object_search_section_size(obj, name, &d_size); 738 if (!ret) 739 *size = d_size; 740 } 741 742 return *size ? 0 : ret; 743 } 744 745 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 746 __u32 *off) 747 { 748 Elf_Data *symbols = obj->efile.symbols; 749 const char *sname; 750 size_t si; 751 752 if (!name || !off) 753 return -EINVAL; 754 755 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 756 GElf_Sym sym; 757 758 if (!gelf_getsym(symbols, si, &sym)) 759 continue; 760 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 761 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 762 continue; 763 764 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 765 sym.st_name); 766 if (!sname) { 767 pr_warning("failed to get sym name string for var %s\n", 768 name); 769 return -EIO; 770 } 771 if (strcmp(name, sname) == 0) { 772 *off = sym.st_value; 773 return 0; 774 } 775 } 776 777 return -ENOENT; 778 } 779 780 static bool bpf_object__has_maps(const struct bpf_object *obj) 781 { 782 return obj->efile.maps_shndx >= 0 || 783 obj->efile.data_shndx >= 0 || 784 obj->efile.rodata_shndx >= 0 || 785 obj->efile.bss_shndx >= 0; 786 } 787 788 static int 789 bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map, 790 enum libbpf_map_type type, Elf_Data *data, 791 void **data_buff) 792 { 793 struct bpf_map_def *def = &map->def; 794 char map_name[BPF_OBJ_NAME_LEN]; 795 796 map->libbpf_type = type; 797 map->offset = ~(typeof(map->offset))0; 798 snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, 799 libbpf_type_to_btf_name[type]); 800 map->name = strdup(map_name); 801 if (!map->name) { 802 pr_warning("failed to alloc map name\n"); 803 return -ENOMEM; 804 } 805 806 def->type = BPF_MAP_TYPE_ARRAY; 807 def->key_size = sizeof(int); 808 def->value_size = data->d_size; 809 def->max_entries = 1; 810 def->map_flags = type == LIBBPF_MAP_RODATA ? 811 BPF_F_RDONLY_PROG : 0; 812 if (data_buff) { 813 *data_buff = malloc(data->d_size); 814 if (!*data_buff) { 815 zfree(&map->name); 816 pr_warning("failed to alloc map content buffer\n"); 817 return -ENOMEM; 818 } 819 memcpy(*data_buff, data->d_buf, data->d_size); 820 } 821 822 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 823 return 0; 824 } 825 826 static int 827 bpf_object__init_maps(struct bpf_object *obj, int flags) 828 { 829 int i, map_idx, map_def_sz = 0, nr_syms, nr_maps = 0, nr_maps_glob = 0; 830 bool strict = !(flags & MAPS_RELAX_COMPAT); 831 Elf_Data *symbols = obj->efile.symbols; 832 Elf_Data *data = NULL; 833 int ret = 0; 834 835 if (!symbols) 836 return -EINVAL; 837 nr_syms = symbols->d_size / sizeof(GElf_Sym); 838 839 if (obj->efile.maps_shndx >= 0) { 840 Elf_Scn *scn = elf_getscn(obj->efile.elf, 841 obj->efile.maps_shndx); 842 843 if (scn) 844 data = elf_getdata(scn, NULL); 845 if (!scn || !data) { 846 pr_warning("failed to get Elf_Data from map section %d\n", 847 obj->efile.maps_shndx); 848 return -EINVAL; 849 } 850 } 851 852 /* 853 * Count number of maps. Each map has a name. 854 * Array of maps is not supported: only the first element is 855 * considered. 856 * 857 * TODO: Detect array of map and report error. 858 */ 859 if (obj->caps.global_data) { 860 if (obj->efile.data_shndx >= 0) 861 nr_maps_glob++; 862 if (obj->efile.rodata_shndx >= 0) 863 nr_maps_glob++; 864 if (obj->efile.bss_shndx >= 0) 865 nr_maps_glob++; 866 } 867 868 for (i = 0; data && i < nr_syms; i++) { 869 GElf_Sym sym; 870 871 if (!gelf_getsym(symbols, i, &sym)) 872 continue; 873 if (sym.st_shndx != obj->efile.maps_shndx) 874 continue; 875 nr_maps++; 876 } 877 878 if (!nr_maps && !nr_maps_glob) 879 return 0; 880 881 /* Assume equally sized map definitions */ 882 if (data) { 883 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 884 nr_maps, data->d_size); 885 886 map_def_sz = data->d_size / nr_maps; 887 if (!data->d_size || (data->d_size % nr_maps) != 0) { 888 pr_warning("unable to determine map definition size " 889 "section %s, %d maps in %zd bytes\n", 890 obj->path, nr_maps, data->d_size); 891 return -EINVAL; 892 } 893 } 894 895 nr_maps += nr_maps_glob; 896 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 897 if (!obj->maps) { 898 pr_warning("alloc maps for object failed\n"); 899 return -ENOMEM; 900 } 901 obj->nr_maps = nr_maps; 902 903 for (i = 0; i < nr_maps; i++) { 904 /* 905 * fill all fd with -1 so won't close incorrect 906 * fd (fd=0 is stdin) when failure (zclose won't close 907 * negative fd)). 908 */ 909 obj->maps[i].fd = -1; 910 obj->maps[i].inner_map_fd = -1; 911 } 912 913 /* 914 * Fill obj->maps using data in "maps" section. 915 */ 916 for (i = 0, map_idx = 0; data && i < nr_syms; i++) { 917 GElf_Sym sym; 918 const char *map_name; 919 struct bpf_map_def *def; 920 921 if (!gelf_getsym(symbols, i, &sym)) 922 continue; 923 if (sym.st_shndx != obj->efile.maps_shndx) 924 continue; 925 926 map_name = elf_strptr(obj->efile.elf, 927 obj->efile.strtabidx, 928 sym.st_name); 929 930 obj->maps[map_idx].libbpf_type = LIBBPF_MAP_UNSPEC; 931 obj->maps[map_idx].offset = sym.st_value; 932 if (sym.st_value + map_def_sz > data->d_size) { 933 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 934 obj->path, map_name); 935 return -EINVAL; 936 } 937 938 obj->maps[map_idx].name = strdup(map_name); 939 if (!obj->maps[map_idx].name) { 940 pr_warning("failed to alloc map name\n"); 941 return -ENOMEM; 942 } 943 pr_debug("map %d is \"%s\"\n", map_idx, 944 obj->maps[map_idx].name); 945 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 946 /* 947 * If the definition of the map in the object file fits in 948 * bpf_map_def, copy it. Any extra fields in our version 949 * of bpf_map_def will default to zero as a result of the 950 * calloc above. 951 */ 952 if (map_def_sz <= sizeof(struct bpf_map_def)) { 953 memcpy(&obj->maps[map_idx].def, def, map_def_sz); 954 } else { 955 /* 956 * Here the map structure being read is bigger than what 957 * we expect, truncate if the excess bits are all zero. 958 * If they are not zero, reject this map as 959 * incompatible. 960 */ 961 char *b; 962 for (b = ((char *)def) + sizeof(struct bpf_map_def); 963 b < ((char *)def) + map_def_sz; b++) { 964 if (*b != 0) { 965 pr_warning("maps section in %s: \"%s\" " 966 "has unrecognized, non-zero " 967 "options\n", 968 obj->path, map_name); 969 if (strict) 970 return -EINVAL; 971 } 972 } 973 memcpy(&obj->maps[map_idx].def, def, 974 sizeof(struct bpf_map_def)); 975 } 976 map_idx++; 977 } 978 979 if (!obj->caps.global_data) 980 goto finalize; 981 982 /* 983 * Populate rest of obj->maps with libbpf internal maps. 984 */ 985 if (obj->efile.data_shndx >= 0) 986 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 987 LIBBPF_MAP_DATA, 988 obj->efile.data, 989 &obj->sections.data); 990 if (!ret && obj->efile.rodata_shndx >= 0) 991 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 992 LIBBPF_MAP_RODATA, 993 obj->efile.rodata, 994 &obj->sections.rodata); 995 if (!ret && obj->efile.bss_shndx >= 0) 996 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 997 LIBBPF_MAP_BSS, 998 obj->efile.bss, NULL); 999 finalize: 1000 if (!ret) 1001 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), 1002 compare_bpf_map); 1003 return ret; 1004 } 1005 1006 static bool section_have_execinstr(struct bpf_object *obj, int idx) 1007 { 1008 Elf_Scn *scn; 1009 GElf_Shdr sh; 1010 1011 scn = elf_getscn(obj->efile.elf, idx); 1012 if (!scn) 1013 return false; 1014 1015 if (gelf_getshdr(scn, &sh) != &sh) 1016 return false; 1017 1018 if (sh.sh_flags & SHF_EXECINSTR) 1019 return true; 1020 1021 return false; 1022 } 1023 1024 static int bpf_object__elf_collect(struct bpf_object *obj, int flags) 1025 { 1026 Elf *elf = obj->efile.elf; 1027 GElf_Ehdr *ep = &obj->efile.ehdr; 1028 Elf_Data *btf_ext_data = NULL; 1029 Elf_Data *btf_data = NULL; 1030 Elf_Scn *scn = NULL; 1031 int idx = 0, err = 0; 1032 1033 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1034 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 1035 pr_warning("failed to get e_shstrndx from %s\n", 1036 obj->path); 1037 return -LIBBPF_ERRNO__FORMAT; 1038 } 1039 1040 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1041 char *name; 1042 GElf_Shdr sh; 1043 Elf_Data *data; 1044 1045 idx++; 1046 if (gelf_getshdr(scn, &sh) != &sh) { 1047 pr_warning("failed to get section(%d) header from %s\n", 1048 idx, obj->path); 1049 err = -LIBBPF_ERRNO__FORMAT; 1050 goto out; 1051 } 1052 1053 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 1054 if (!name) { 1055 pr_warning("failed to get section(%d) name from %s\n", 1056 idx, obj->path); 1057 err = -LIBBPF_ERRNO__FORMAT; 1058 goto out; 1059 } 1060 1061 data = elf_getdata(scn, 0); 1062 if (!data) { 1063 pr_warning("failed to get section(%d) data from %s(%s)\n", 1064 idx, name, obj->path); 1065 err = -LIBBPF_ERRNO__FORMAT; 1066 goto out; 1067 } 1068 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 1069 idx, name, (unsigned long)data->d_size, 1070 (int)sh.sh_link, (unsigned long)sh.sh_flags, 1071 (int)sh.sh_type); 1072 1073 if (strcmp(name, "license") == 0) { 1074 err = bpf_object__init_license(obj, 1075 data->d_buf, 1076 data->d_size); 1077 } else if (strcmp(name, "version") == 0) { 1078 err = bpf_object__init_kversion(obj, 1079 data->d_buf, 1080 data->d_size); 1081 } else if (strcmp(name, "maps") == 0) { 1082 obj->efile.maps_shndx = idx; 1083 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 1084 btf_data = data; 1085 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 1086 btf_ext_data = data; 1087 } else if (sh.sh_type == SHT_SYMTAB) { 1088 if (obj->efile.symbols) { 1089 pr_warning("bpf: multiple SYMTAB in %s\n", 1090 obj->path); 1091 err = -LIBBPF_ERRNO__FORMAT; 1092 } else { 1093 obj->efile.symbols = data; 1094 obj->efile.strtabidx = sh.sh_link; 1095 } 1096 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 1097 if (sh.sh_flags & SHF_EXECINSTR) { 1098 if (strcmp(name, ".text") == 0) 1099 obj->efile.text_shndx = idx; 1100 err = bpf_object__add_program(obj, data->d_buf, 1101 data->d_size, name, idx); 1102 if (err) { 1103 char errmsg[STRERR_BUFSIZE]; 1104 char *cp = libbpf_strerror_r(-err, errmsg, 1105 sizeof(errmsg)); 1106 1107 pr_warning("failed to alloc program %s (%s): %s", 1108 name, obj->path, cp); 1109 } 1110 } else if (strcmp(name, ".data") == 0) { 1111 obj->efile.data = data; 1112 obj->efile.data_shndx = idx; 1113 } else if (strcmp(name, ".rodata") == 0) { 1114 obj->efile.rodata = data; 1115 obj->efile.rodata_shndx = idx; 1116 } else { 1117 pr_debug("skip section(%d) %s\n", idx, name); 1118 } 1119 } else if (sh.sh_type == SHT_REL) { 1120 void *reloc = obj->efile.reloc; 1121 int nr_reloc = obj->efile.nr_reloc + 1; 1122 int sec = sh.sh_info; /* points to other section */ 1123 1124 /* Only do relo for section with exec instructions */ 1125 if (!section_have_execinstr(obj, sec)) { 1126 pr_debug("skip relo %s(%d) for section(%d)\n", 1127 name, idx, sec); 1128 continue; 1129 } 1130 1131 reloc = reallocarray(reloc, nr_reloc, 1132 sizeof(*obj->efile.reloc)); 1133 if (!reloc) { 1134 pr_warning("realloc failed\n"); 1135 err = -ENOMEM; 1136 } else { 1137 int n = nr_reloc - 1; 1138 1139 obj->efile.reloc = reloc; 1140 obj->efile.nr_reloc = nr_reloc; 1141 1142 obj->efile.reloc[n].shdr = sh; 1143 obj->efile.reloc[n].data = data; 1144 } 1145 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { 1146 obj->efile.bss = data; 1147 obj->efile.bss_shndx = idx; 1148 } else { 1149 pr_debug("skip section(%d) %s\n", idx, name); 1150 } 1151 if (err) 1152 goto out; 1153 } 1154 1155 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 1156 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 1157 return LIBBPF_ERRNO__FORMAT; 1158 } 1159 if (btf_data) { 1160 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 1161 if (IS_ERR(obj->btf)) { 1162 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1163 BTF_ELF_SEC, PTR_ERR(obj->btf)); 1164 obj->btf = NULL; 1165 } else { 1166 err = btf__finalize_data(obj, obj->btf); 1167 if (!err) 1168 err = btf__load(obj->btf); 1169 if (err) { 1170 pr_warning("Error finalizing and loading %s into kernel: %d. Ignored and continue.\n", 1171 BTF_ELF_SEC, err); 1172 btf__free(obj->btf); 1173 obj->btf = NULL; 1174 err = 0; 1175 } 1176 } 1177 } 1178 if (btf_ext_data) { 1179 if (!obj->btf) { 1180 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 1181 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 1182 } else { 1183 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 1184 btf_ext_data->d_size); 1185 if (IS_ERR(obj->btf_ext)) { 1186 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1187 BTF_EXT_ELF_SEC, 1188 PTR_ERR(obj->btf_ext)); 1189 obj->btf_ext = NULL; 1190 } 1191 } 1192 } 1193 if (bpf_object__has_maps(obj)) { 1194 err = bpf_object__init_maps(obj, flags); 1195 if (err) 1196 goto out; 1197 } 1198 err = bpf_object__init_prog_names(obj); 1199 out: 1200 return err; 1201 } 1202 1203 static struct bpf_program * 1204 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 1205 { 1206 struct bpf_program *prog; 1207 size_t i; 1208 1209 for (i = 0; i < obj->nr_programs; i++) { 1210 prog = &obj->programs[i]; 1211 if (prog->idx == idx) 1212 return prog; 1213 } 1214 return NULL; 1215 } 1216 1217 struct bpf_program * 1218 bpf_object__find_program_by_title(struct bpf_object *obj, const char *title) 1219 { 1220 struct bpf_program *pos; 1221 1222 bpf_object__for_each_program(pos, obj) { 1223 if (pos->section_name && !strcmp(pos->section_name, title)) 1224 return pos; 1225 } 1226 return NULL; 1227 } 1228 1229 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 1230 int shndx) 1231 { 1232 return shndx == obj->efile.data_shndx || 1233 shndx == obj->efile.bss_shndx || 1234 shndx == obj->efile.rodata_shndx; 1235 } 1236 1237 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 1238 int shndx) 1239 { 1240 return shndx == obj->efile.maps_shndx; 1241 } 1242 1243 static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, 1244 int shndx) 1245 { 1246 return shndx == obj->efile.text_shndx || 1247 bpf_object__shndx_is_maps(obj, shndx) || 1248 bpf_object__shndx_is_data(obj, shndx); 1249 } 1250 1251 static enum libbpf_map_type 1252 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 1253 { 1254 if (shndx == obj->efile.data_shndx) 1255 return LIBBPF_MAP_DATA; 1256 else if (shndx == obj->efile.bss_shndx) 1257 return LIBBPF_MAP_BSS; 1258 else if (shndx == obj->efile.rodata_shndx) 1259 return LIBBPF_MAP_RODATA; 1260 else 1261 return LIBBPF_MAP_UNSPEC; 1262 } 1263 1264 static int 1265 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 1266 Elf_Data *data, struct bpf_object *obj) 1267 { 1268 Elf_Data *symbols = obj->efile.symbols; 1269 struct bpf_map *maps = obj->maps; 1270 size_t nr_maps = obj->nr_maps; 1271 int i, nrels; 1272 1273 pr_debug("collecting relocating info for: '%s'\n", 1274 prog->section_name); 1275 nrels = shdr->sh_size / shdr->sh_entsize; 1276 1277 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 1278 if (!prog->reloc_desc) { 1279 pr_warning("failed to alloc memory in relocation\n"); 1280 return -ENOMEM; 1281 } 1282 prog->nr_reloc = nrels; 1283 1284 for (i = 0; i < nrels; i++) { 1285 GElf_Sym sym; 1286 GElf_Rel rel; 1287 unsigned int insn_idx; 1288 unsigned int shdr_idx; 1289 struct bpf_insn *insns = prog->insns; 1290 enum libbpf_map_type type; 1291 const char *name; 1292 size_t map_idx; 1293 1294 if (!gelf_getrel(data, i, &rel)) { 1295 pr_warning("relocation: failed to get %d reloc\n", i); 1296 return -LIBBPF_ERRNO__FORMAT; 1297 } 1298 1299 if (!gelf_getsym(symbols, 1300 GELF_R_SYM(rel.r_info), 1301 &sym)) { 1302 pr_warning("relocation: symbol %"PRIx64" not found\n", 1303 GELF_R_SYM(rel.r_info)); 1304 return -LIBBPF_ERRNO__FORMAT; 1305 } 1306 1307 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1308 sym.st_name) ? : "<?>"; 1309 1310 pr_debug("relo for %lld value %lld name %d (\'%s\')\n", 1311 (long long) (rel.r_info >> 32), 1312 (long long) sym.st_value, sym.st_name, name); 1313 1314 shdr_idx = sym.st_shndx; 1315 if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { 1316 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", 1317 prog->section_name, shdr_idx); 1318 return -LIBBPF_ERRNO__RELOC; 1319 } 1320 1321 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 1322 pr_debug("relocation: insn_idx=%u\n", insn_idx); 1323 1324 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 1325 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 1326 pr_warning("incorrect bpf_call opcode\n"); 1327 return -LIBBPF_ERRNO__RELOC; 1328 } 1329 prog->reloc_desc[i].type = RELO_CALL; 1330 prog->reloc_desc[i].insn_idx = insn_idx; 1331 prog->reloc_desc[i].text_off = sym.st_value; 1332 obj->has_pseudo_calls = true; 1333 continue; 1334 } 1335 1336 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 1337 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 1338 insn_idx, insns[insn_idx].code); 1339 return -LIBBPF_ERRNO__RELOC; 1340 } 1341 1342 if (bpf_object__shndx_is_maps(obj, shdr_idx) || 1343 bpf_object__shndx_is_data(obj, shdr_idx)) { 1344 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 1345 if (type != LIBBPF_MAP_UNSPEC) { 1346 if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { 1347 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", 1348 name, insn_idx, insns[insn_idx].code); 1349 return -LIBBPF_ERRNO__RELOC; 1350 } 1351 if (!obj->caps.global_data) { 1352 pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", 1353 name, insn_idx); 1354 return -LIBBPF_ERRNO__RELOC; 1355 } 1356 } 1357 1358 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 1359 if (maps[map_idx].libbpf_type != type) 1360 continue; 1361 if (type != LIBBPF_MAP_UNSPEC || 1362 (type == LIBBPF_MAP_UNSPEC && 1363 maps[map_idx].offset == sym.st_value)) { 1364 pr_debug("relocation: find map %zd (%s) for insn %u\n", 1365 map_idx, maps[map_idx].name, insn_idx); 1366 break; 1367 } 1368 } 1369 1370 if (map_idx >= nr_maps) { 1371 pr_warning("bpf relocation: map_idx %d large than %d\n", 1372 (int)map_idx, (int)nr_maps - 1); 1373 return -LIBBPF_ERRNO__RELOC; 1374 } 1375 1376 prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? 1377 RELO_DATA : RELO_LD64; 1378 prog->reloc_desc[i].insn_idx = insn_idx; 1379 prog->reloc_desc[i].map_idx = map_idx; 1380 } 1381 } 1382 return 0; 1383 } 1384 1385 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) 1386 { 1387 struct bpf_map_def *def = &map->def; 1388 __u32 key_type_id = 0, value_type_id = 0; 1389 int ret; 1390 1391 if (!bpf_map__is_internal(map)) { 1392 ret = btf__get_map_kv_tids(btf, map->name, def->key_size, 1393 def->value_size, &key_type_id, 1394 &value_type_id); 1395 } else { 1396 /* 1397 * LLVM annotates global data differently in BTF, that is, 1398 * only as '.data', '.bss' or '.rodata'. 1399 */ 1400 ret = btf__find_by_name(btf, 1401 libbpf_type_to_btf_name[map->libbpf_type]); 1402 } 1403 if (ret < 0) 1404 return ret; 1405 1406 map->btf_key_type_id = key_type_id; 1407 map->btf_value_type_id = bpf_map__is_internal(map) ? 1408 ret : value_type_id; 1409 return 0; 1410 } 1411 1412 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 1413 { 1414 struct bpf_map_info info = {}; 1415 __u32 len = sizeof(info); 1416 int new_fd, err; 1417 char *new_name; 1418 1419 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1420 if (err) 1421 return err; 1422 1423 new_name = strdup(info.name); 1424 if (!new_name) 1425 return -errno; 1426 1427 new_fd = open("/", O_RDONLY | O_CLOEXEC); 1428 if (new_fd < 0) 1429 goto err_free_new_name; 1430 1431 new_fd = dup3(fd, new_fd, O_CLOEXEC); 1432 if (new_fd < 0) 1433 goto err_close_new_fd; 1434 1435 err = zclose(map->fd); 1436 if (err) 1437 goto err_close_new_fd; 1438 free(map->name); 1439 1440 map->fd = new_fd; 1441 map->name = new_name; 1442 map->def.type = info.type; 1443 map->def.key_size = info.key_size; 1444 map->def.value_size = info.value_size; 1445 map->def.max_entries = info.max_entries; 1446 map->def.map_flags = info.map_flags; 1447 map->btf_key_type_id = info.btf_key_type_id; 1448 map->btf_value_type_id = info.btf_value_type_id; 1449 1450 return 0; 1451 1452 err_close_new_fd: 1453 close(new_fd); 1454 err_free_new_name: 1455 free(new_name); 1456 return -errno; 1457 } 1458 1459 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 1460 { 1461 if (!map || !max_entries) 1462 return -EINVAL; 1463 1464 /* If map already created, its attributes can't be changed. */ 1465 if (map->fd >= 0) 1466 return -EBUSY; 1467 1468 map->def.max_entries = max_entries; 1469 1470 return 0; 1471 } 1472 1473 static int 1474 bpf_object__probe_name(struct bpf_object *obj) 1475 { 1476 struct bpf_load_program_attr attr; 1477 char *cp, errmsg[STRERR_BUFSIZE]; 1478 struct bpf_insn insns[] = { 1479 BPF_MOV64_IMM(BPF_REG_0, 0), 1480 BPF_EXIT_INSN(), 1481 }; 1482 int ret; 1483 1484 /* make sure basic loading works */ 1485 1486 memset(&attr, 0, sizeof(attr)); 1487 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1488 attr.insns = insns; 1489 attr.insns_cnt = ARRAY_SIZE(insns); 1490 attr.license = "GPL"; 1491 1492 ret = bpf_load_program_xattr(&attr, NULL, 0); 1493 if (ret < 0) { 1494 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1495 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", 1496 __func__, cp, errno); 1497 return -errno; 1498 } 1499 close(ret); 1500 1501 /* now try the same program, but with the name */ 1502 1503 attr.name = "test"; 1504 ret = bpf_load_program_xattr(&attr, NULL, 0); 1505 if (ret >= 0) { 1506 obj->caps.name = 1; 1507 close(ret); 1508 } 1509 1510 return 0; 1511 } 1512 1513 static int 1514 bpf_object__probe_global_data(struct bpf_object *obj) 1515 { 1516 struct bpf_load_program_attr prg_attr; 1517 struct bpf_create_map_attr map_attr; 1518 char *cp, errmsg[STRERR_BUFSIZE]; 1519 struct bpf_insn insns[] = { 1520 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 1521 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 1522 BPF_MOV64_IMM(BPF_REG_0, 0), 1523 BPF_EXIT_INSN(), 1524 }; 1525 int ret, map; 1526 1527 memset(&map_attr, 0, sizeof(map_attr)); 1528 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 1529 map_attr.key_size = sizeof(int); 1530 map_attr.value_size = 32; 1531 map_attr.max_entries = 1; 1532 1533 map = bpf_create_map_xattr(&map_attr); 1534 if (map < 0) { 1535 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1536 pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", 1537 __func__, cp, errno); 1538 return -errno; 1539 } 1540 1541 insns[0].imm = map; 1542 1543 memset(&prg_attr, 0, sizeof(prg_attr)); 1544 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1545 prg_attr.insns = insns; 1546 prg_attr.insns_cnt = ARRAY_SIZE(insns); 1547 prg_attr.license = "GPL"; 1548 1549 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 1550 if (ret >= 0) { 1551 obj->caps.global_data = 1; 1552 close(ret); 1553 } 1554 1555 close(map); 1556 return 0; 1557 } 1558 1559 static int 1560 bpf_object__probe_caps(struct bpf_object *obj) 1561 { 1562 int (*probe_fn[])(struct bpf_object *obj) = { 1563 bpf_object__probe_name, 1564 bpf_object__probe_global_data, 1565 }; 1566 int i, ret; 1567 1568 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { 1569 ret = probe_fn[i](obj); 1570 if (ret < 0) 1571 return ret; 1572 } 1573 1574 return 0; 1575 } 1576 1577 static int 1578 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 1579 { 1580 char *cp, errmsg[STRERR_BUFSIZE]; 1581 int err, zero = 0; 1582 __u8 *data; 1583 1584 /* Nothing to do here since kernel already zero-initializes .bss map. */ 1585 if (map->libbpf_type == LIBBPF_MAP_BSS) 1586 return 0; 1587 1588 data = map->libbpf_type == LIBBPF_MAP_DATA ? 1589 obj->sections.data : obj->sections.rodata; 1590 1591 err = bpf_map_update_elem(map->fd, &zero, data, 0); 1592 /* Freeze .rodata map as read-only from syscall side. */ 1593 if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { 1594 err = bpf_map_freeze(map->fd); 1595 if (err) { 1596 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1597 pr_warning("Error freezing map(%s) as read-only: %s\n", 1598 map->name, cp); 1599 err = 0; 1600 } 1601 } 1602 return err; 1603 } 1604 1605 static int 1606 bpf_object__create_maps(struct bpf_object *obj) 1607 { 1608 struct bpf_create_map_attr create_attr = {}; 1609 unsigned int i; 1610 int err; 1611 1612 for (i = 0; i < obj->nr_maps; i++) { 1613 struct bpf_map *map = &obj->maps[i]; 1614 struct bpf_map_def *def = &map->def; 1615 char *cp, errmsg[STRERR_BUFSIZE]; 1616 int *pfd = &map->fd; 1617 1618 if (map->fd >= 0) { 1619 pr_debug("skip map create (preset) %s: fd=%d\n", 1620 map->name, map->fd); 1621 continue; 1622 } 1623 1624 if (obj->caps.name) 1625 create_attr.name = map->name; 1626 create_attr.map_ifindex = map->map_ifindex; 1627 create_attr.map_type = def->type; 1628 create_attr.map_flags = def->map_flags; 1629 create_attr.key_size = def->key_size; 1630 create_attr.value_size = def->value_size; 1631 create_attr.max_entries = def->max_entries; 1632 create_attr.btf_fd = 0; 1633 create_attr.btf_key_type_id = 0; 1634 create_attr.btf_value_type_id = 0; 1635 if (bpf_map_type__is_map_in_map(def->type) && 1636 map->inner_map_fd >= 0) 1637 create_attr.inner_map_fd = map->inner_map_fd; 1638 1639 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) { 1640 create_attr.btf_fd = btf__fd(obj->btf); 1641 create_attr.btf_key_type_id = map->btf_key_type_id; 1642 create_attr.btf_value_type_id = map->btf_value_type_id; 1643 } 1644 1645 *pfd = bpf_create_map_xattr(&create_attr); 1646 if (*pfd < 0 && create_attr.btf_key_type_id) { 1647 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1648 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 1649 map->name, cp, errno); 1650 create_attr.btf_fd = 0; 1651 create_attr.btf_key_type_id = 0; 1652 create_attr.btf_value_type_id = 0; 1653 map->btf_key_type_id = 0; 1654 map->btf_value_type_id = 0; 1655 *pfd = bpf_create_map_xattr(&create_attr); 1656 } 1657 1658 if (*pfd < 0) { 1659 size_t j; 1660 1661 err = *pfd; 1662 err_out: 1663 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1664 pr_warning("failed to create map (name: '%s'): %s\n", 1665 map->name, cp); 1666 for (j = 0; j < i; j++) 1667 zclose(obj->maps[j].fd); 1668 return err; 1669 } 1670 1671 if (bpf_map__is_internal(map)) { 1672 err = bpf_object__populate_internal_map(obj, map); 1673 if (err < 0) { 1674 zclose(*pfd); 1675 goto err_out; 1676 } 1677 } 1678 1679 pr_debug("create map %s: fd=%d\n", map->name, *pfd); 1680 } 1681 1682 return 0; 1683 } 1684 1685 static int 1686 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 1687 void *btf_prog_info, const char *info_name) 1688 { 1689 if (err != -ENOENT) { 1690 pr_warning("Error in loading %s for sec %s.\n", 1691 info_name, prog->section_name); 1692 return err; 1693 } 1694 1695 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 1696 1697 if (btf_prog_info) { 1698 /* 1699 * Some info has already been found but has problem 1700 * in the last btf_ext reloc. Must have to error 1701 * out. 1702 */ 1703 pr_warning("Error in relocating %s for sec %s.\n", 1704 info_name, prog->section_name); 1705 return err; 1706 } 1707 1708 /* 1709 * Have problem loading the very first info. Ignore 1710 * the rest. 1711 */ 1712 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", 1713 info_name, prog->section_name, info_name); 1714 return 0; 1715 } 1716 1717 static int 1718 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 1719 const char *section_name, __u32 insn_offset) 1720 { 1721 int err; 1722 1723 if (!insn_offset || prog->func_info) { 1724 /* 1725 * !insn_offset => main program 1726 * 1727 * For sub prog, the main program's func_info has to 1728 * be loaded first (i.e. prog->func_info != NULL) 1729 */ 1730 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 1731 section_name, insn_offset, 1732 &prog->func_info, 1733 &prog->func_info_cnt); 1734 if (err) 1735 return check_btf_ext_reloc_err(prog, err, 1736 prog->func_info, 1737 "bpf_func_info"); 1738 1739 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 1740 } 1741 1742 if (!insn_offset || prog->line_info) { 1743 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 1744 section_name, insn_offset, 1745 &prog->line_info, 1746 &prog->line_info_cnt); 1747 if (err) 1748 return check_btf_ext_reloc_err(prog, err, 1749 prog->line_info, 1750 "bpf_line_info"); 1751 1752 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 1753 } 1754 1755 if (!insn_offset) 1756 prog->btf_fd = btf__fd(obj->btf); 1757 1758 return 0; 1759 } 1760 1761 static int 1762 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 1763 struct reloc_desc *relo) 1764 { 1765 struct bpf_insn *insn, *new_insn; 1766 struct bpf_program *text; 1767 size_t new_cnt; 1768 int err; 1769 1770 if (relo->type != RELO_CALL) 1771 return -LIBBPF_ERRNO__RELOC; 1772 1773 if (prog->idx == obj->efile.text_shndx) { 1774 pr_warning("relo in .text insn %d into off %d\n", 1775 relo->insn_idx, relo->text_off); 1776 return -LIBBPF_ERRNO__RELOC; 1777 } 1778 1779 if (prog->main_prog_cnt == 0) { 1780 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 1781 if (!text) { 1782 pr_warning("no .text section found yet relo into text exist\n"); 1783 return -LIBBPF_ERRNO__RELOC; 1784 } 1785 new_cnt = prog->insns_cnt + text->insns_cnt; 1786 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 1787 if (!new_insn) { 1788 pr_warning("oom in prog realloc\n"); 1789 return -ENOMEM; 1790 } 1791 1792 if (obj->btf_ext) { 1793 err = bpf_program_reloc_btf_ext(prog, obj, 1794 text->section_name, 1795 prog->insns_cnt); 1796 if (err) 1797 return err; 1798 } 1799 1800 memcpy(new_insn + prog->insns_cnt, text->insns, 1801 text->insns_cnt * sizeof(*insn)); 1802 prog->insns = new_insn; 1803 prog->main_prog_cnt = prog->insns_cnt; 1804 prog->insns_cnt = new_cnt; 1805 pr_debug("added %zd insn from %s to prog %s\n", 1806 text->insns_cnt, text->section_name, 1807 prog->section_name); 1808 } 1809 insn = &prog->insns[relo->insn_idx]; 1810 insn->imm += prog->main_prog_cnt - relo->insn_idx; 1811 return 0; 1812 } 1813 1814 static int 1815 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1816 { 1817 int i, err; 1818 1819 if (!prog) 1820 return 0; 1821 1822 if (obj->btf_ext) { 1823 err = bpf_program_reloc_btf_ext(prog, obj, 1824 prog->section_name, 0); 1825 if (err) 1826 return err; 1827 } 1828 1829 if (!prog->reloc_desc) 1830 return 0; 1831 1832 for (i = 0; i < prog->nr_reloc; i++) { 1833 if (prog->reloc_desc[i].type == RELO_LD64 || 1834 prog->reloc_desc[i].type == RELO_DATA) { 1835 bool relo_data = prog->reloc_desc[i].type == RELO_DATA; 1836 struct bpf_insn *insns = prog->insns; 1837 int insn_idx, map_idx; 1838 1839 insn_idx = prog->reloc_desc[i].insn_idx; 1840 map_idx = prog->reloc_desc[i].map_idx; 1841 1842 if (insn_idx + 1 >= (int)prog->insns_cnt) { 1843 pr_warning("relocation out of range: '%s'\n", 1844 prog->section_name); 1845 return -LIBBPF_ERRNO__RELOC; 1846 } 1847 1848 if (!relo_data) { 1849 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1850 } else { 1851 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; 1852 insns[insn_idx + 1].imm = insns[insn_idx].imm; 1853 } 1854 insns[insn_idx].imm = obj->maps[map_idx].fd; 1855 } else if (prog->reloc_desc[i].type == RELO_CALL) { 1856 err = bpf_program__reloc_text(prog, obj, 1857 &prog->reloc_desc[i]); 1858 if (err) 1859 return err; 1860 } 1861 } 1862 1863 zfree(&prog->reloc_desc); 1864 prog->nr_reloc = 0; 1865 return 0; 1866 } 1867 1868 1869 static int 1870 bpf_object__relocate(struct bpf_object *obj) 1871 { 1872 struct bpf_program *prog; 1873 size_t i; 1874 int err; 1875 1876 for (i = 0; i < obj->nr_programs; i++) { 1877 prog = &obj->programs[i]; 1878 1879 err = bpf_program__relocate(prog, obj); 1880 if (err) { 1881 pr_warning("failed to relocate '%s'\n", 1882 prog->section_name); 1883 return err; 1884 } 1885 } 1886 return 0; 1887 } 1888 1889 static int bpf_object__collect_reloc(struct bpf_object *obj) 1890 { 1891 int i, err; 1892 1893 if (!obj_elf_valid(obj)) { 1894 pr_warning("Internal error: elf object is closed\n"); 1895 return -LIBBPF_ERRNO__INTERNAL; 1896 } 1897 1898 for (i = 0; i < obj->efile.nr_reloc; i++) { 1899 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 1900 Elf_Data *data = obj->efile.reloc[i].data; 1901 int idx = shdr->sh_info; 1902 struct bpf_program *prog; 1903 1904 if (shdr->sh_type != SHT_REL) { 1905 pr_warning("internal error at %d\n", __LINE__); 1906 return -LIBBPF_ERRNO__INTERNAL; 1907 } 1908 1909 prog = bpf_object__find_prog_by_idx(obj, idx); 1910 if (!prog) { 1911 pr_warning("relocation failed: no section(%d)\n", idx); 1912 return -LIBBPF_ERRNO__RELOC; 1913 } 1914 1915 err = bpf_program__collect_reloc(prog, 1916 shdr, data, 1917 obj); 1918 if (err) 1919 return err; 1920 } 1921 return 0; 1922 } 1923 1924 static int 1925 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 1926 char *license, __u32 kern_version, int *pfd) 1927 { 1928 struct bpf_load_program_attr load_attr; 1929 char *cp, errmsg[STRERR_BUFSIZE]; 1930 int log_buf_size = BPF_LOG_BUF_SIZE; 1931 char *log_buf; 1932 int ret; 1933 1934 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 1935 load_attr.prog_type = prog->type; 1936 load_attr.expected_attach_type = prog->expected_attach_type; 1937 if (prog->caps->name) 1938 load_attr.name = prog->name; 1939 load_attr.insns = insns; 1940 load_attr.insns_cnt = insns_cnt; 1941 load_attr.license = license; 1942 load_attr.kern_version = kern_version; 1943 load_attr.prog_ifindex = prog->prog_ifindex; 1944 load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; 1945 load_attr.func_info = prog->func_info; 1946 load_attr.func_info_rec_size = prog->func_info_rec_size; 1947 load_attr.func_info_cnt = prog->func_info_cnt; 1948 load_attr.line_info = prog->line_info; 1949 load_attr.line_info_rec_size = prog->line_info_rec_size; 1950 load_attr.line_info_cnt = prog->line_info_cnt; 1951 load_attr.log_level = prog->log_level; 1952 if (!load_attr.insns || !load_attr.insns_cnt) 1953 return -EINVAL; 1954 1955 retry_load: 1956 log_buf = malloc(log_buf_size); 1957 if (!log_buf) 1958 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 1959 1960 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); 1961 1962 if (ret >= 0) { 1963 if (load_attr.log_level) 1964 pr_debug("verifier log:\n%s", log_buf); 1965 *pfd = ret; 1966 ret = 0; 1967 goto out; 1968 } 1969 1970 if (errno == ENOSPC) { 1971 log_buf_size <<= 1; 1972 free(log_buf); 1973 goto retry_load; 1974 } 1975 ret = -LIBBPF_ERRNO__LOAD; 1976 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1977 pr_warning("load bpf program failed: %s\n", cp); 1978 1979 if (log_buf && log_buf[0] != '\0') { 1980 ret = -LIBBPF_ERRNO__VERIFY; 1981 pr_warning("-- BEGIN DUMP LOG ---\n"); 1982 pr_warning("\n%s\n", log_buf); 1983 pr_warning("-- END LOG --\n"); 1984 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 1985 pr_warning("Program too large (%zu insns), at most %d insns\n", 1986 load_attr.insns_cnt, BPF_MAXINSNS); 1987 ret = -LIBBPF_ERRNO__PROG2BIG; 1988 } else { 1989 /* Wrong program type? */ 1990 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 1991 int fd; 1992 1993 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 1994 load_attr.expected_attach_type = 0; 1995 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 1996 if (fd >= 0) { 1997 close(fd); 1998 ret = -LIBBPF_ERRNO__PROGTYPE; 1999 goto out; 2000 } 2001 } 2002 2003 if (log_buf) 2004 ret = -LIBBPF_ERRNO__KVER; 2005 } 2006 2007 out: 2008 free(log_buf); 2009 return ret; 2010 } 2011 2012 int 2013 bpf_program__load(struct bpf_program *prog, 2014 char *license, __u32 kern_version) 2015 { 2016 int err = 0, fd, i; 2017 2018 if (prog->instances.nr < 0 || !prog->instances.fds) { 2019 if (prog->preprocessor) { 2020 pr_warning("Internal error: can't load program '%s'\n", 2021 prog->section_name); 2022 return -LIBBPF_ERRNO__INTERNAL; 2023 } 2024 2025 prog->instances.fds = malloc(sizeof(int)); 2026 if (!prog->instances.fds) { 2027 pr_warning("Not enough memory for BPF fds\n"); 2028 return -ENOMEM; 2029 } 2030 prog->instances.nr = 1; 2031 prog->instances.fds[0] = -1; 2032 } 2033 2034 if (!prog->preprocessor) { 2035 if (prog->instances.nr != 1) { 2036 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 2037 prog->section_name, prog->instances.nr); 2038 } 2039 err = load_program(prog, prog->insns, prog->insns_cnt, 2040 license, kern_version, &fd); 2041 if (!err) 2042 prog->instances.fds[0] = fd; 2043 goto out; 2044 } 2045 2046 for (i = 0; i < prog->instances.nr; i++) { 2047 struct bpf_prog_prep_result result; 2048 bpf_program_prep_t preprocessor = prog->preprocessor; 2049 2050 memset(&result, 0, sizeof(result)); 2051 err = preprocessor(prog, i, prog->insns, 2052 prog->insns_cnt, &result); 2053 if (err) { 2054 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 2055 i, prog->section_name); 2056 goto out; 2057 } 2058 2059 if (!result.new_insn_ptr || !result.new_insn_cnt) { 2060 pr_debug("Skip loading the %dth instance of program '%s'\n", 2061 i, prog->section_name); 2062 prog->instances.fds[i] = -1; 2063 if (result.pfd) 2064 *result.pfd = -1; 2065 continue; 2066 } 2067 2068 err = load_program(prog, result.new_insn_ptr, 2069 result.new_insn_cnt, 2070 license, kern_version, &fd); 2071 2072 if (err) { 2073 pr_warning("Loading the %dth instance of program '%s' failed\n", 2074 i, prog->section_name); 2075 goto out; 2076 } 2077 2078 if (result.pfd) 2079 *result.pfd = fd; 2080 prog->instances.fds[i] = fd; 2081 } 2082 out: 2083 if (err) 2084 pr_warning("failed to load program '%s'\n", 2085 prog->section_name); 2086 zfree(&prog->insns); 2087 prog->insns_cnt = 0; 2088 return err; 2089 } 2090 2091 static bool bpf_program__is_function_storage(struct bpf_program *prog, 2092 struct bpf_object *obj) 2093 { 2094 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 2095 } 2096 2097 static int 2098 bpf_object__load_progs(struct bpf_object *obj) 2099 { 2100 size_t i; 2101 int err; 2102 2103 for (i = 0; i < obj->nr_programs; i++) { 2104 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 2105 continue; 2106 err = bpf_program__load(&obj->programs[i], 2107 obj->license, 2108 obj->kern_version); 2109 if (err) 2110 return err; 2111 } 2112 return 0; 2113 } 2114 2115 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) 2116 { 2117 switch (type) { 2118 case BPF_PROG_TYPE_SOCKET_FILTER: 2119 case BPF_PROG_TYPE_SCHED_CLS: 2120 case BPF_PROG_TYPE_SCHED_ACT: 2121 case BPF_PROG_TYPE_XDP: 2122 case BPF_PROG_TYPE_CGROUP_SKB: 2123 case BPF_PROG_TYPE_CGROUP_SOCK: 2124 case BPF_PROG_TYPE_LWT_IN: 2125 case BPF_PROG_TYPE_LWT_OUT: 2126 case BPF_PROG_TYPE_LWT_XMIT: 2127 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 2128 case BPF_PROG_TYPE_SOCK_OPS: 2129 case BPF_PROG_TYPE_SK_SKB: 2130 case BPF_PROG_TYPE_CGROUP_DEVICE: 2131 case BPF_PROG_TYPE_SK_MSG: 2132 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2133 case BPF_PROG_TYPE_LIRC_MODE2: 2134 case BPF_PROG_TYPE_SK_REUSEPORT: 2135 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2136 case BPF_PROG_TYPE_UNSPEC: 2137 case BPF_PROG_TYPE_TRACEPOINT: 2138 case BPF_PROG_TYPE_RAW_TRACEPOINT: 2139 case BPF_PROG_TYPE_PERF_EVENT: 2140 case BPF_PROG_TYPE_CGROUP_SYSCTL: 2141 return false; 2142 case BPF_PROG_TYPE_KPROBE: 2143 default: 2144 return true; 2145 } 2146 } 2147 2148 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) 2149 { 2150 if (needs_kver && obj->kern_version == 0) { 2151 pr_warning("%s doesn't provide kernel version\n", 2152 obj->path); 2153 return -LIBBPF_ERRNO__KVERSION; 2154 } 2155 return 0; 2156 } 2157 2158 static struct bpf_object * 2159 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, 2160 bool needs_kver, int flags) 2161 { 2162 struct bpf_object *obj; 2163 int err; 2164 2165 if (elf_version(EV_CURRENT) == EV_NONE) { 2166 pr_warning("failed to init libelf for %s\n", path); 2167 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 2168 } 2169 2170 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 2171 if (IS_ERR(obj)) 2172 return obj; 2173 2174 CHECK_ERR(bpf_object__elf_init(obj), err, out); 2175 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 2176 CHECK_ERR(bpf_object__probe_caps(obj), err, out); 2177 CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); 2178 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 2179 CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); 2180 2181 bpf_object__elf_finish(obj); 2182 return obj; 2183 out: 2184 bpf_object__close(obj); 2185 return ERR_PTR(err); 2186 } 2187 2188 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, 2189 int flags) 2190 { 2191 /* param validation */ 2192 if (!attr->file) 2193 return NULL; 2194 2195 pr_debug("loading %s\n", attr->file); 2196 2197 return __bpf_object__open(attr->file, NULL, 0, 2198 bpf_prog_type__needs_kver(attr->prog_type), 2199 flags); 2200 } 2201 2202 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 2203 { 2204 return __bpf_object__open_xattr(attr, 0); 2205 } 2206 2207 struct bpf_object *bpf_object__open(const char *path) 2208 { 2209 struct bpf_object_open_attr attr = { 2210 .file = path, 2211 .prog_type = BPF_PROG_TYPE_UNSPEC, 2212 }; 2213 2214 return bpf_object__open_xattr(&attr); 2215 } 2216 2217 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 2218 size_t obj_buf_sz, 2219 const char *name) 2220 { 2221 char tmp_name[64]; 2222 2223 /* param validation */ 2224 if (!obj_buf || obj_buf_sz <= 0) 2225 return NULL; 2226 2227 if (!name) { 2228 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 2229 (unsigned long)obj_buf, 2230 (unsigned long)obj_buf_sz); 2231 tmp_name[sizeof(tmp_name) - 1] = '\0'; 2232 name = tmp_name; 2233 } 2234 pr_debug("loading object '%s' from buffer\n", 2235 name); 2236 2237 return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); 2238 } 2239 2240 int bpf_object__unload(struct bpf_object *obj) 2241 { 2242 size_t i; 2243 2244 if (!obj) 2245 return -EINVAL; 2246 2247 for (i = 0; i < obj->nr_maps; i++) 2248 zclose(obj->maps[i].fd); 2249 2250 for (i = 0; i < obj->nr_programs; i++) 2251 bpf_program__unload(&obj->programs[i]); 2252 2253 return 0; 2254 } 2255 2256 int bpf_object__load(struct bpf_object *obj) 2257 { 2258 int err; 2259 2260 if (!obj) 2261 return -EINVAL; 2262 2263 if (obj->loaded) { 2264 pr_warning("object should not be loaded twice\n"); 2265 return -EINVAL; 2266 } 2267 2268 obj->loaded = true; 2269 2270 CHECK_ERR(bpf_object__create_maps(obj), err, out); 2271 CHECK_ERR(bpf_object__relocate(obj), err, out); 2272 CHECK_ERR(bpf_object__load_progs(obj), err, out); 2273 2274 return 0; 2275 out: 2276 bpf_object__unload(obj); 2277 pr_warning("failed to load object '%s'\n", obj->path); 2278 return err; 2279 } 2280 2281 static int check_path(const char *path) 2282 { 2283 char *cp, errmsg[STRERR_BUFSIZE]; 2284 struct statfs st_fs; 2285 char *dname, *dir; 2286 int err = 0; 2287 2288 if (path == NULL) 2289 return -EINVAL; 2290 2291 dname = strdup(path); 2292 if (dname == NULL) 2293 return -ENOMEM; 2294 2295 dir = dirname(dname); 2296 if (statfs(dir, &st_fs)) { 2297 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2298 pr_warning("failed to statfs %s: %s\n", dir, cp); 2299 err = -errno; 2300 } 2301 free(dname); 2302 2303 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 2304 pr_warning("specified path %s is not on BPF FS\n", path); 2305 err = -EINVAL; 2306 } 2307 2308 return err; 2309 } 2310 2311 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 2312 int instance) 2313 { 2314 char *cp, errmsg[STRERR_BUFSIZE]; 2315 int err; 2316 2317 err = check_path(path); 2318 if (err) 2319 return err; 2320 2321 if (prog == NULL) { 2322 pr_warning("invalid program pointer\n"); 2323 return -EINVAL; 2324 } 2325 2326 if (instance < 0 || instance >= prog->instances.nr) { 2327 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2328 instance, prog->section_name, prog->instances.nr); 2329 return -EINVAL; 2330 } 2331 2332 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 2333 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2334 pr_warning("failed to pin program: %s\n", cp); 2335 return -errno; 2336 } 2337 pr_debug("pinned program '%s'\n", path); 2338 2339 return 0; 2340 } 2341 2342 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 2343 int instance) 2344 { 2345 int err; 2346 2347 err = check_path(path); 2348 if (err) 2349 return err; 2350 2351 if (prog == NULL) { 2352 pr_warning("invalid program pointer\n"); 2353 return -EINVAL; 2354 } 2355 2356 if (instance < 0 || instance >= prog->instances.nr) { 2357 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2358 instance, prog->section_name, prog->instances.nr); 2359 return -EINVAL; 2360 } 2361 2362 err = unlink(path); 2363 if (err != 0) 2364 return -errno; 2365 pr_debug("unpinned program '%s'\n", path); 2366 2367 return 0; 2368 } 2369 2370 static int make_dir(const char *path) 2371 { 2372 char *cp, errmsg[STRERR_BUFSIZE]; 2373 int err = 0; 2374 2375 if (mkdir(path, 0700) && errno != EEXIST) 2376 err = -errno; 2377 2378 if (err) { 2379 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 2380 pr_warning("failed to mkdir %s: %s\n", path, cp); 2381 } 2382 return err; 2383 } 2384 2385 int bpf_program__pin(struct bpf_program *prog, const char *path) 2386 { 2387 int i, err; 2388 2389 err = check_path(path); 2390 if (err) 2391 return err; 2392 2393 if (prog == NULL) { 2394 pr_warning("invalid program pointer\n"); 2395 return -EINVAL; 2396 } 2397 2398 if (prog->instances.nr <= 0) { 2399 pr_warning("no instances of prog %s to pin\n", 2400 prog->section_name); 2401 return -EINVAL; 2402 } 2403 2404 if (prog->instances.nr == 1) { 2405 /* don't create subdirs when pinning single instance */ 2406 return bpf_program__pin_instance(prog, path, 0); 2407 } 2408 2409 err = make_dir(path); 2410 if (err) 2411 return err; 2412 2413 for (i = 0; i < prog->instances.nr; i++) { 2414 char buf[PATH_MAX]; 2415 int len; 2416 2417 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2418 if (len < 0) { 2419 err = -EINVAL; 2420 goto err_unpin; 2421 } else if (len >= PATH_MAX) { 2422 err = -ENAMETOOLONG; 2423 goto err_unpin; 2424 } 2425 2426 err = bpf_program__pin_instance(prog, buf, i); 2427 if (err) 2428 goto err_unpin; 2429 } 2430 2431 return 0; 2432 2433 err_unpin: 2434 for (i = i - 1; i >= 0; i--) { 2435 char buf[PATH_MAX]; 2436 int len; 2437 2438 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2439 if (len < 0) 2440 continue; 2441 else if (len >= PATH_MAX) 2442 continue; 2443 2444 bpf_program__unpin_instance(prog, buf, i); 2445 } 2446 2447 rmdir(path); 2448 2449 return err; 2450 } 2451 2452 int bpf_program__unpin(struct bpf_program *prog, const char *path) 2453 { 2454 int i, err; 2455 2456 err = check_path(path); 2457 if (err) 2458 return err; 2459 2460 if (prog == NULL) { 2461 pr_warning("invalid program pointer\n"); 2462 return -EINVAL; 2463 } 2464 2465 if (prog->instances.nr <= 0) { 2466 pr_warning("no instances of prog %s to pin\n", 2467 prog->section_name); 2468 return -EINVAL; 2469 } 2470 2471 if (prog->instances.nr == 1) { 2472 /* don't create subdirs when pinning single instance */ 2473 return bpf_program__unpin_instance(prog, path, 0); 2474 } 2475 2476 for (i = 0; i < prog->instances.nr; i++) { 2477 char buf[PATH_MAX]; 2478 int len; 2479 2480 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2481 if (len < 0) 2482 return -EINVAL; 2483 else if (len >= PATH_MAX) 2484 return -ENAMETOOLONG; 2485 2486 err = bpf_program__unpin_instance(prog, buf, i); 2487 if (err) 2488 return err; 2489 } 2490 2491 err = rmdir(path); 2492 if (err) 2493 return -errno; 2494 2495 return 0; 2496 } 2497 2498 int bpf_map__pin(struct bpf_map *map, const char *path) 2499 { 2500 char *cp, errmsg[STRERR_BUFSIZE]; 2501 int err; 2502 2503 err = check_path(path); 2504 if (err) 2505 return err; 2506 2507 if (map == NULL) { 2508 pr_warning("invalid map pointer\n"); 2509 return -EINVAL; 2510 } 2511 2512 if (bpf_obj_pin(map->fd, path)) { 2513 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2514 pr_warning("failed to pin map: %s\n", cp); 2515 return -errno; 2516 } 2517 2518 pr_debug("pinned map '%s'\n", path); 2519 2520 return 0; 2521 } 2522 2523 int bpf_map__unpin(struct bpf_map *map, const char *path) 2524 { 2525 int err; 2526 2527 err = check_path(path); 2528 if (err) 2529 return err; 2530 2531 if (map == NULL) { 2532 pr_warning("invalid map pointer\n"); 2533 return -EINVAL; 2534 } 2535 2536 err = unlink(path); 2537 if (err != 0) 2538 return -errno; 2539 pr_debug("unpinned map '%s'\n", path); 2540 2541 return 0; 2542 } 2543 2544 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 2545 { 2546 struct bpf_map *map; 2547 int err; 2548 2549 if (!obj) 2550 return -ENOENT; 2551 2552 if (!obj->loaded) { 2553 pr_warning("object not yet loaded; load it first\n"); 2554 return -ENOENT; 2555 } 2556 2557 err = make_dir(path); 2558 if (err) 2559 return err; 2560 2561 bpf_object__for_each_map(map, obj) { 2562 char buf[PATH_MAX]; 2563 int len; 2564 2565 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2566 bpf_map__name(map)); 2567 if (len < 0) { 2568 err = -EINVAL; 2569 goto err_unpin_maps; 2570 } else if (len >= PATH_MAX) { 2571 err = -ENAMETOOLONG; 2572 goto err_unpin_maps; 2573 } 2574 2575 err = bpf_map__pin(map, buf); 2576 if (err) 2577 goto err_unpin_maps; 2578 } 2579 2580 return 0; 2581 2582 err_unpin_maps: 2583 while ((map = bpf_map__prev(map, obj))) { 2584 char buf[PATH_MAX]; 2585 int len; 2586 2587 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2588 bpf_map__name(map)); 2589 if (len < 0) 2590 continue; 2591 else if (len >= PATH_MAX) 2592 continue; 2593 2594 bpf_map__unpin(map, buf); 2595 } 2596 2597 return err; 2598 } 2599 2600 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 2601 { 2602 struct bpf_map *map; 2603 int err; 2604 2605 if (!obj) 2606 return -ENOENT; 2607 2608 bpf_object__for_each_map(map, obj) { 2609 char buf[PATH_MAX]; 2610 int len; 2611 2612 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2613 bpf_map__name(map)); 2614 if (len < 0) 2615 return -EINVAL; 2616 else if (len >= PATH_MAX) 2617 return -ENAMETOOLONG; 2618 2619 err = bpf_map__unpin(map, buf); 2620 if (err) 2621 return err; 2622 } 2623 2624 return 0; 2625 } 2626 2627 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 2628 { 2629 struct bpf_program *prog; 2630 int err; 2631 2632 if (!obj) 2633 return -ENOENT; 2634 2635 if (!obj->loaded) { 2636 pr_warning("object not yet loaded; load it first\n"); 2637 return -ENOENT; 2638 } 2639 2640 err = make_dir(path); 2641 if (err) 2642 return err; 2643 2644 bpf_object__for_each_program(prog, obj) { 2645 char buf[PATH_MAX]; 2646 int len; 2647 2648 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2649 prog->pin_name); 2650 if (len < 0) { 2651 err = -EINVAL; 2652 goto err_unpin_programs; 2653 } else if (len >= PATH_MAX) { 2654 err = -ENAMETOOLONG; 2655 goto err_unpin_programs; 2656 } 2657 2658 err = bpf_program__pin(prog, buf); 2659 if (err) 2660 goto err_unpin_programs; 2661 } 2662 2663 return 0; 2664 2665 err_unpin_programs: 2666 while ((prog = bpf_program__prev(prog, obj))) { 2667 char buf[PATH_MAX]; 2668 int len; 2669 2670 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2671 prog->pin_name); 2672 if (len < 0) 2673 continue; 2674 else if (len >= PATH_MAX) 2675 continue; 2676 2677 bpf_program__unpin(prog, buf); 2678 } 2679 2680 return err; 2681 } 2682 2683 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 2684 { 2685 struct bpf_program *prog; 2686 int err; 2687 2688 if (!obj) 2689 return -ENOENT; 2690 2691 bpf_object__for_each_program(prog, obj) { 2692 char buf[PATH_MAX]; 2693 int len; 2694 2695 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2696 prog->pin_name); 2697 if (len < 0) 2698 return -EINVAL; 2699 else if (len >= PATH_MAX) 2700 return -ENAMETOOLONG; 2701 2702 err = bpf_program__unpin(prog, buf); 2703 if (err) 2704 return err; 2705 } 2706 2707 return 0; 2708 } 2709 2710 int bpf_object__pin(struct bpf_object *obj, const char *path) 2711 { 2712 int err; 2713 2714 err = bpf_object__pin_maps(obj, path); 2715 if (err) 2716 return err; 2717 2718 err = bpf_object__pin_programs(obj, path); 2719 if (err) { 2720 bpf_object__unpin_maps(obj, path); 2721 return err; 2722 } 2723 2724 return 0; 2725 } 2726 2727 void bpf_object__close(struct bpf_object *obj) 2728 { 2729 size_t i; 2730 2731 if (!obj) 2732 return; 2733 2734 if (obj->clear_priv) 2735 obj->clear_priv(obj, obj->priv); 2736 2737 bpf_object__elf_finish(obj); 2738 bpf_object__unload(obj); 2739 btf__free(obj->btf); 2740 btf_ext__free(obj->btf_ext); 2741 2742 for (i = 0; i < obj->nr_maps; i++) { 2743 zfree(&obj->maps[i].name); 2744 if (obj->maps[i].clear_priv) 2745 obj->maps[i].clear_priv(&obj->maps[i], 2746 obj->maps[i].priv); 2747 obj->maps[i].priv = NULL; 2748 obj->maps[i].clear_priv = NULL; 2749 } 2750 2751 zfree(&obj->sections.rodata); 2752 zfree(&obj->sections.data); 2753 zfree(&obj->maps); 2754 obj->nr_maps = 0; 2755 2756 if (obj->programs && obj->nr_programs) { 2757 for (i = 0; i < obj->nr_programs; i++) 2758 bpf_program__exit(&obj->programs[i]); 2759 } 2760 zfree(&obj->programs); 2761 2762 list_del(&obj->list); 2763 free(obj); 2764 } 2765 2766 struct bpf_object * 2767 bpf_object__next(struct bpf_object *prev) 2768 { 2769 struct bpf_object *next; 2770 2771 if (!prev) 2772 next = list_first_entry(&bpf_objects_list, 2773 struct bpf_object, 2774 list); 2775 else 2776 next = list_next_entry(prev, list); 2777 2778 /* Empty list is noticed here so don't need checking on entry. */ 2779 if (&next->list == &bpf_objects_list) 2780 return NULL; 2781 2782 return next; 2783 } 2784 2785 const char *bpf_object__name(struct bpf_object *obj) 2786 { 2787 return obj ? obj->path : ERR_PTR(-EINVAL); 2788 } 2789 2790 unsigned int bpf_object__kversion(struct bpf_object *obj) 2791 { 2792 return obj ? obj->kern_version : 0; 2793 } 2794 2795 struct btf *bpf_object__btf(struct bpf_object *obj) 2796 { 2797 return obj ? obj->btf : NULL; 2798 } 2799 2800 int bpf_object__btf_fd(const struct bpf_object *obj) 2801 { 2802 return obj->btf ? btf__fd(obj->btf) : -1; 2803 } 2804 2805 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 2806 bpf_object_clear_priv_t clear_priv) 2807 { 2808 if (obj->priv && obj->clear_priv) 2809 obj->clear_priv(obj, obj->priv); 2810 2811 obj->priv = priv; 2812 obj->clear_priv = clear_priv; 2813 return 0; 2814 } 2815 2816 void *bpf_object__priv(struct bpf_object *obj) 2817 { 2818 return obj ? obj->priv : ERR_PTR(-EINVAL); 2819 } 2820 2821 static struct bpf_program * 2822 __bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward) 2823 { 2824 size_t nr_programs = obj->nr_programs; 2825 ssize_t idx; 2826 2827 if (!nr_programs) 2828 return NULL; 2829 2830 if (!p) 2831 /* Iter from the beginning */ 2832 return forward ? &obj->programs[0] : 2833 &obj->programs[nr_programs - 1]; 2834 2835 if (p->obj != obj) { 2836 pr_warning("error: program handler doesn't match object\n"); 2837 return NULL; 2838 } 2839 2840 idx = (p - obj->programs) + (forward ? 1 : -1); 2841 if (idx >= obj->nr_programs || idx < 0) 2842 return NULL; 2843 return &obj->programs[idx]; 2844 } 2845 2846 struct bpf_program * 2847 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 2848 { 2849 struct bpf_program *prog = prev; 2850 2851 do { 2852 prog = __bpf_program__iter(prog, obj, true); 2853 } while (prog && bpf_program__is_function_storage(prog, obj)); 2854 2855 return prog; 2856 } 2857 2858 struct bpf_program * 2859 bpf_program__prev(struct bpf_program *next, struct bpf_object *obj) 2860 { 2861 struct bpf_program *prog = next; 2862 2863 do { 2864 prog = __bpf_program__iter(prog, obj, false); 2865 } while (prog && bpf_program__is_function_storage(prog, obj)); 2866 2867 return prog; 2868 } 2869 2870 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 2871 bpf_program_clear_priv_t clear_priv) 2872 { 2873 if (prog->priv && prog->clear_priv) 2874 prog->clear_priv(prog, prog->priv); 2875 2876 prog->priv = priv; 2877 prog->clear_priv = clear_priv; 2878 return 0; 2879 } 2880 2881 void *bpf_program__priv(struct bpf_program *prog) 2882 { 2883 return prog ? prog->priv : ERR_PTR(-EINVAL); 2884 } 2885 2886 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 2887 { 2888 prog->prog_ifindex = ifindex; 2889 } 2890 2891 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 2892 { 2893 const char *title; 2894 2895 title = prog->section_name; 2896 if (needs_copy) { 2897 title = strdup(title); 2898 if (!title) { 2899 pr_warning("failed to strdup program title\n"); 2900 return ERR_PTR(-ENOMEM); 2901 } 2902 } 2903 2904 return title; 2905 } 2906 2907 int bpf_program__fd(struct bpf_program *prog) 2908 { 2909 return bpf_program__nth_fd(prog, 0); 2910 } 2911 2912 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 2913 bpf_program_prep_t prep) 2914 { 2915 int *instances_fds; 2916 2917 if (nr_instances <= 0 || !prep) 2918 return -EINVAL; 2919 2920 if (prog->instances.nr > 0 || prog->instances.fds) { 2921 pr_warning("Can't set pre-processor after loading\n"); 2922 return -EINVAL; 2923 } 2924 2925 instances_fds = malloc(sizeof(int) * nr_instances); 2926 if (!instances_fds) { 2927 pr_warning("alloc memory failed for fds\n"); 2928 return -ENOMEM; 2929 } 2930 2931 /* fill all fd with -1 */ 2932 memset(instances_fds, -1, sizeof(int) * nr_instances); 2933 2934 prog->instances.nr = nr_instances; 2935 prog->instances.fds = instances_fds; 2936 prog->preprocessor = prep; 2937 return 0; 2938 } 2939 2940 int bpf_program__nth_fd(struct bpf_program *prog, int n) 2941 { 2942 int fd; 2943 2944 if (!prog) 2945 return -EINVAL; 2946 2947 if (n >= prog->instances.nr || n < 0) { 2948 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 2949 n, prog->section_name, prog->instances.nr); 2950 return -EINVAL; 2951 } 2952 2953 fd = prog->instances.fds[n]; 2954 if (fd < 0) { 2955 pr_warning("%dth instance of program '%s' is invalid\n", 2956 n, prog->section_name); 2957 return -ENOENT; 2958 } 2959 2960 return fd; 2961 } 2962 2963 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 2964 { 2965 prog->type = type; 2966 } 2967 2968 static bool bpf_program__is_type(struct bpf_program *prog, 2969 enum bpf_prog_type type) 2970 { 2971 return prog ? (prog->type == type) : false; 2972 } 2973 2974 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 2975 int bpf_program__set_##NAME(struct bpf_program *prog) \ 2976 { \ 2977 if (!prog) \ 2978 return -EINVAL; \ 2979 bpf_program__set_type(prog, TYPE); \ 2980 return 0; \ 2981 } \ 2982 \ 2983 bool bpf_program__is_##NAME(struct bpf_program *prog) \ 2984 { \ 2985 return bpf_program__is_type(prog, TYPE); \ 2986 } \ 2987 2988 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 2989 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 2990 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 2991 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 2992 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 2993 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 2994 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 2995 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 2996 2997 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 2998 enum bpf_attach_type type) 2999 { 3000 prog->expected_attach_type = type; 3001 } 3002 3003 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ 3004 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } 3005 3006 /* Programs that can NOT be attached. */ 3007 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) 3008 3009 /* Programs that can be attached. */ 3010 #define BPF_APROG_SEC(string, ptype, atype) \ 3011 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) 3012 3013 /* Programs that must specify expected attach type at load time. */ 3014 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 3015 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) 3016 3017 /* Programs that can be attached but attach type can't be identified by section 3018 * name. Kept for backward compatibility. 3019 */ 3020 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 3021 3022 static const struct { 3023 const char *sec; 3024 size_t len; 3025 enum bpf_prog_type prog_type; 3026 enum bpf_attach_type expected_attach_type; 3027 int is_attachable; 3028 enum bpf_attach_type attach_type; 3029 } section_names[] = { 3030 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 3031 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 3032 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 3033 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 3034 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 3035 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 3036 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), 3037 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 3038 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 3039 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 3040 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 3041 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 3042 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 3043 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 3044 BPF_CGROUP_INET_INGRESS), 3045 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 3046 BPF_CGROUP_INET_EGRESS), 3047 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 3048 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 3049 BPF_CGROUP_INET_SOCK_CREATE), 3050 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 3051 BPF_CGROUP_INET4_POST_BIND), 3052 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 3053 BPF_CGROUP_INET6_POST_BIND), 3054 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 3055 BPF_CGROUP_DEVICE), 3056 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 3057 BPF_CGROUP_SOCK_OPS), 3058 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 3059 BPF_SK_SKB_STREAM_PARSER), 3060 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 3061 BPF_SK_SKB_STREAM_VERDICT), 3062 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 3063 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 3064 BPF_SK_MSG_VERDICT), 3065 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 3066 BPF_LIRC_MODE2), 3067 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 3068 BPF_FLOW_DISSECTOR), 3069 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3070 BPF_CGROUP_INET4_BIND), 3071 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3072 BPF_CGROUP_INET6_BIND), 3073 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3074 BPF_CGROUP_INET4_CONNECT), 3075 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3076 BPF_CGROUP_INET6_CONNECT), 3077 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3078 BPF_CGROUP_UDP4_SENDMSG), 3079 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3080 BPF_CGROUP_UDP6_SENDMSG), 3081 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 3082 BPF_CGROUP_SYSCTL), 3083 }; 3084 3085 #undef BPF_PROG_SEC_IMPL 3086 #undef BPF_PROG_SEC 3087 #undef BPF_APROG_SEC 3088 #undef BPF_EAPROG_SEC 3089 #undef BPF_APROG_COMPAT 3090 3091 #define MAX_TYPE_NAME_SIZE 32 3092 3093 static char *libbpf_get_type_names(bool attach_type) 3094 { 3095 int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; 3096 char *buf; 3097 3098 buf = malloc(len); 3099 if (!buf) 3100 return NULL; 3101 3102 buf[0] = '\0'; 3103 /* Forge string buf with all available names */ 3104 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3105 if (attach_type && !section_names[i].is_attachable) 3106 continue; 3107 3108 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { 3109 free(buf); 3110 return NULL; 3111 } 3112 strcat(buf, " "); 3113 strcat(buf, section_names[i].sec); 3114 } 3115 3116 return buf; 3117 } 3118 3119 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 3120 enum bpf_attach_type *expected_attach_type) 3121 { 3122 char *type_names; 3123 int i; 3124 3125 if (!name) 3126 return -EINVAL; 3127 3128 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3129 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3130 continue; 3131 *prog_type = section_names[i].prog_type; 3132 *expected_attach_type = section_names[i].expected_attach_type; 3133 return 0; 3134 } 3135 pr_warning("failed to guess program type based on ELF section name '%s'\n", name); 3136 type_names = libbpf_get_type_names(false); 3137 if (type_names != NULL) { 3138 pr_info("supported section(type) names are:%s\n", type_names); 3139 free(type_names); 3140 } 3141 3142 return -EINVAL; 3143 } 3144 3145 int libbpf_attach_type_by_name(const char *name, 3146 enum bpf_attach_type *attach_type) 3147 { 3148 char *type_names; 3149 int i; 3150 3151 if (!name) 3152 return -EINVAL; 3153 3154 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3155 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3156 continue; 3157 if (!section_names[i].is_attachable) 3158 return -EINVAL; 3159 *attach_type = section_names[i].attach_type; 3160 return 0; 3161 } 3162 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); 3163 type_names = libbpf_get_type_names(true); 3164 if (type_names != NULL) { 3165 pr_info("attachable section(type) names are:%s\n", type_names); 3166 free(type_names); 3167 } 3168 3169 return -EINVAL; 3170 } 3171 3172 static int 3173 bpf_program__identify_section(struct bpf_program *prog, 3174 enum bpf_prog_type *prog_type, 3175 enum bpf_attach_type *expected_attach_type) 3176 { 3177 return libbpf_prog_type_by_name(prog->section_name, prog_type, 3178 expected_attach_type); 3179 } 3180 3181 int bpf_map__fd(struct bpf_map *map) 3182 { 3183 return map ? map->fd : -EINVAL; 3184 } 3185 3186 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 3187 { 3188 return map ? &map->def : ERR_PTR(-EINVAL); 3189 } 3190 3191 const char *bpf_map__name(struct bpf_map *map) 3192 { 3193 return map ? map->name : NULL; 3194 } 3195 3196 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 3197 { 3198 return map ? map->btf_key_type_id : 0; 3199 } 3200 3201 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 3202 { 3203 return map ? map->btf_value_type_id : 0; 3204 } 3205 3206 int bpf_map__set_priv(struct bpf_map *map, void *priv, 3207 bpf_map_clear_priv_t clear_priv) 3208 { 3209 if (!map) 3210 return -EINVAL; 3211 3212 if (map->priv) { 3213 if (map->clear_priv) 3214 map->clear_priv(map, map->priv); 3215 } 3216 3217 map->priv = priv; 3218 map->clear_priv = clear_priv; 3219 return 0; 3220 } 3221 3222 void *bpf_map__priv(struct bpf_map *map) 3223 { 3224 return map ? map->priv : ERR_PTR(-EINVAL); 3225 } 3226 3227 bool bpf_map__is_offload_neutral(struct bpf_map *map) 3228 { 3229 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 3230 } 3231 3232 bool bpf_map__is_internal(struct bpf_map *map) 3233 { 3234 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 3235 } 3236 3237 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 3238 { 3239 map->map_ifindex = ifindex; 3240 } 3241 3242 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 3243 { 3244 if (!bpf_map_type__is_map_in_map(map->def.type)) { 3245 pr_warning("error: unsupported map type\n"); 3246 return -EINVAL; 3247 } 3248 if (map->inner_map_fd != -1) { 3249 pr_warning("error: inner_map_fd already specified\n"); 3250 return -EINVAL; 3251 } 3252 map->inner_map_fd = fd; 3253 return 0; 3254 } 3255 3256 static struct bpf_map * 3257 __bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i) 3258 { 3259 ssize_t idx; 3260 struct bpf_map *s, *e; 3261 3262 if (!obj || !obj->maps) 3263 return NULL; 3264 3265 s = obj->maps; 3266 e = obj->maps + obj->nr_maps; 3267 3268 if ((m < s) || (m >= e)) { 3269 pr_warning("error in %s: map handler doesn't belong to object\n", 3270 __func__); 3271 return NULL; 3272 } 3273 3274 idx = (m - obj->maps) + i; 3275 if (idx >= obj->nr_maps || idx < 0) 3276 return NULL; 3277 return &obj->maps[idx]; 3278 } 3279 3280 struct bpf_map * 3281 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 3282 { 3283 if (prev == NULL) 3284 return obj->maps; 3285 3286 return __bpf_map__iter(prev, obj, 1); 3287 } 3288 3289 struct bpf_map * 3290 bpf_map__prev(struct bpf_map *next, struct bpf_object *obj) 3291 { 3292 if (next == NULL) { 3293 if (!obj->nr_maps) 3294 return NULL; 3295 return obj->maps + obj->nr_maps - 1; 3296 } 3297 3298 return __bpf_map__iter(next, obj, -1); 3299 } 3300 3301 struct bpf_map * 3302 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 3303 { 3304 struct bpf_map *pos; 3305 3306 bpf_object__for_each_map(pos, obj) { 3307 if (pos->name && !strcmp(pos->name, name)) 3308 return pos; 3309 } 3310 return NULL; 3311 } 3312 3313 int 3314 bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name) 3315 { 3316 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 3317 } 3318 3319 struct bpf_map * 3320 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 3321 { 3322 int i; 3323 3324 for (i = 0; i < obj->nr_maps; i++) { 3325 if (obj->maps[i].offset == offset) 3326 return &obj->maps[i]; 3327 } 3328 return ERR_PTR(-ENOENT); 3329 } 3330 3331 long libbpf_get_error(const void *ptr) 3332 { 3333 if (IS_ERR(ptr)) 3334 return PTR_ERR(ptr); 3335 return 0; 3336 } 3337 3338 int bpf_prog_load(const char *file, enum bpf_prog_type type, 3339 struct bpf_object **pobj, int *prog_fd) 3340 { 3341 struct bpf_prog_load_attr attr; 3342 3343 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 3344 attr.file = file; 3345 attr.prog_type = type; 3346 attr.expected_attach_type = 0; 3347 3348 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 3349 } 3350 3351 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 3352 struct bpf_object **pobj, int *prog_fd) 3353 { 3354 struct bpf_object_open_attr open_attr = { 3355 .file = attr->file, 3356 .prog_type = attr->prog_type, 3357 }; 3358 struct bpf_program *prog, *first_prog = NULL; 3359 enum bpf_attach_type expected_attach_type; 3360 enum bpf_prog_type prog_type; 3361 struct bpf_object *obj; 3362 struct bpf_map *map; 3363 int err; 3364 3365 if (!attr) 3366 return -EINVAL; 3367 if (!attr->file) 3368 return -EINVAL; 3369 3370 obj = bpf_object__open_xattr(&open_attr); 3371 if (IS_ERR_OR_NULL(obj)) 3372 return -ENOENT; 3373 3374 bpf_object__for_each_program(prog, obj) { 3375 /* 3376 * If type is not specified, try to guess it based on 3377 * section name. 3378 */ 3379 prog_type = attr->prog_type; 3380 prog->prog_ifindex = attr->ifindex; 3381 expected_attach_type = attr->expected_attach_type; 3382 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 3383 err = bpf_program__identify_section(prog, &prog_type, 3384 &expected_attach_type); 3385 if (err < 0) { 3386 bpf_object__close(obj); 3387 return -EINVAL; 3388 } 3389 } 3390 3391 bpf_program__set_type(prog, prog_type); 3392 bpf_program__set_expected_attach_type(prog, 3393 expected_attach_type); 3394 3395 prog->log_level = attr->log_level; 3396 if (!first_prog) 3397 first_prog = prog; 3398 } 3399 3400 bpf_object__for_each_map(map, obj) { 3401 if (!bpf_map__is_offload_neutral(map)) 3402 map->map_ifindex = attr->ifindex; 3403 } 3404 3405 if (!first_prog) { 3406 pr_warning("object file doesn't contain bpf program\n"); 3407 bpf_object__close(obj); 3408 return -ENOENT; 3409 } 3410 3411 err = bpf_object__load(obj); 3412 if (err) { 3413 bpf_object__close(obj); 3414 return -EINVAL; 3415 } 3416 3417 *pobj = obj; 3418 *prog_fd = bpf_program__fd(first_prog); 3419 return 0; 3420 } 3421 3422 enum bpf_perf_event_ret 3423 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 3424 void **copy_mem, size_t *copy_size, 3425 bpf_perf_event_print_t fn, void *private_data) 3426 { 3427 struct perf_event_mmap_page *header = mmap_mem; 3428 __u64 data_head = ring_buffer_read_head(header); 3429 __u64 data_tail = header->data_tail; 3430 void *base = ((__u8 *)header) + page_size; 3431 int ret = LIBBPF_PERF_EVENT_CONT; 3432 struct perf_event_header *ehdr; 3433 size_t ehdr_size; 3434 3435 while (data_head != data_tail) { 3436 ehdr = base + (data_tail & (mmap_size - 1)); 3437 ehdr_size = ehdr->size; 3438 3439 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 3440 void *copy_start = ehdr; 3441 size_t len_first = base + mmap_size - copy_start; 3442 size_t len_secnd = ehdr_size - len_first; 3443 3444 if (*copy_size < ehdr_size) { 3445 free(*copy_mem); 3446 *copy_mem = malloc(ehdr_size); 3447 if (!*copy_mem) { 3448 *copy_size = 0; 3449 ret = LIBBPF_PERF_EVENT_ERROR; 3450 break; 3451 } 3452 *copy_size = ehdr_size; 3453 } 3454 3455 memcpy(*copy_mem, copy_start, len_first); 3456 memcpy(*copy_mem + len_first, base, len_secnd); 3457 ehdr = *copy_mem; 3458 } 3459 3460 ret = fn(ehdr, private_data); 3461 data_tail += ehdr_size; 3462 if (ret != LIBBPF_PERF_EVENT_CONT) 3463 break; 3464 } 3465 3466 ring_buffer_write_tail(header, data_tail); 3467 return ret; 3468 } 3469 3470 struct bpf_prog_info_array_desc { 3471 int array_offset; /* e.g. offset of jited_prog_insns */ 3472 int count_offset; /* e.g. offset of jited_prog_len */ 3473 int size_offset; /* > 0: offset of rec size, 3474 * < 0: fix size of -size_offset 3475 */ 3476 }; 3477 3478 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 3479 [BPF_PROG_INFO_JITED_INSNS] = { 3480 offsetof(struct bpf_prog_info, jited_prog_insns), 3481 offsetof(struct bpf_prog_info, jited_prog_len), 3482 -1, 3483 }, 3484 [BPF_PROG_INFO_XLATED_INSNS] = { 3485 offsetof(struct bpf_prog_info, xlated_prog_insns), 3486 offsetof(struct bpf_prog_info, xlated_prog_len), 3487 -1, 3488 }, 3489 [BPF_PROG_INFO_MAP_IDS] = { 3490 offsetof(struct bpf_prog_info, map_ids), 3491 offsetof(struct bpf_prog_info, nr_map_ids), 3492 -(int)sizeof(__u32), 3493 }, 3494 [BPF_PROG_INFO_JITED_KSYMS] = { 3495 offsetof(struct bpf_prog_info, jited_ksyms), 3496 offsetof(struct bpf_prog_info, nr_jited_ksyms), 3497 -(int)sizeof(__u64), 3498 }, 3499 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 3500 offsetof(struct bpf_prog_info, jited_func_lens), 3501 offsetof(struct bpf_prog_info, nr_jited_func_lens), 3502 -(int)sizeof(__u32), 3503 }, 3504 [BPF_PROG_INFO_FUNC_INFO] = { 3505 offsetof(struct bpf_prog_info, func_info), 3506 offsetof(struct bpf_prog_info, nr_func_info), 3507 offsetof(struct bpf_prog_info, func_info_rec_size), 3508 }, 3509 [BPF_PROG_INFO_LINE_INFO] = { 3510 offsetof(struct bpf_prog_info, line_info), 3511 offsetof(struct bpf_prog_info, nr_line_info), 3512 offsetof(struct bpf_prog_info, line_info_rec_size), 3513 }, 3514 [BPF_PROG_INFO_JITED_LINE_INFO] = { 3515 offsetof(struct bpf_prog_info, jited_line_info), 3516 offsetof(struct bpf_prog_info, nr_jited_line_info), 3517 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 3518 }, 3519 [BPF_PROG_INFO_PROG_TAGS] = { 3520 offsetof(struct bpf_prog_info, prog_tags), 3521 offsetof(struct bpf_prog_info, nr_prog_tags), 3522 -(int)sizeof(__u8) * BPF_TAG_SIZE, 3523 }, 3524 3525 }; 3526 3527 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) 3528 { 3529 __u32 *array = (__u32 *)info; 3530 3531 if (offset >= 0) 3532 return array[offset / sizeof(__u32)]; 3533 return -(int)offset; 3534 } 3535 3536 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) 3537 { 3538 __u64 *array = (__u64 *)info; 3539 3540 if (offset >= 0) 3541 return array[offset / sizeof(__u64)]; 3542 return -(int)offset; 3543 } 3544 3545 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 3546 __u32 val) 3547 { 3548 __u32 *array = (__u32 *)info; 3549 3550 if (offset >= 0) 3551 array[offset / sizeof(__u32)] = val; 3552 } 3553 3554 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 3555 __u64 val) 3556 { 3557 __u64 *array = (__u64 *)info; 3558 3559 if (offset >= 0) 3560 array[offset / sizeof(__u64)] = val; 3561 } 3562 3563 struct bpf_prog_info_linear * 3564 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 3565 { 3566 struct bpf_prog_info_linear *info_linear; 3567 struct bpf_prog_info info = {}; 3568 __u32 info_len = sizeof(info); 3569 __u32 data_len = 0; 3570 int i, err; 3571 void *ptr; 3572 3573 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 3574 return ERR_PTR(-EINVAL); 3575 3576 /* step 1: get array dimensions */ 3577 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 3578 if (err) { 3579 pr_debug("can't get prog info: %s", strerror(errno)); 3580 return ERR_PTR(-EFAULT); 3581 } 3582 3583 /* step 2: calculate total size of all arrays */ 3584 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3585 bool include_array = (arrays & (1UL << i)) > 0; 3586 struct bpf_prog_info_array_desc *desc; 3587 __u32 count, size; 3588 3589 desc = bpf_prog_info_array_desc + i; 3590 3591 /* kernel is too old to support this field */ 3592 if (info_len < desc->array_offset + sizeof(__u32) || 3593 info_len < desc->count_offset + sizeof(__u32) || 3594 (desc->size_offset > 0 && info_len < desc->size_offset)) 3595 include_array = false; 3596 3597 if (!include_array) { 3598 arrays &= ~(1UL << i); /* clear the bit */ 3599 continue; 3600 } 3601 3602 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3603 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3604 3605 data_len += count * size; 3606 } 3607 3608 /* step 3: allocate continuous memory */ 3609 data_len = roundup(data_len, sizeof(__u64)); 3610 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 3611 if (!info_linear) 3612 return ERR_PTR(-ENOMEM); 3613 3614 /* step 4: fill data to info_linear->info */ 3615 info_linear->arrays = arrays; 3616 memset(&info_linear->info, 0, sizeof(info)); 3617 ptr = info_linear->data; 3618 3619 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3620 struct bpf_prog_info_array_desc *desc; 3621 __u32 count, size; 3622 3623 if ((arrays & (1UL << i)) == 0) 3624 continue; 3625 3626 desc = bpf_prog_info_array_desc + i; 3627 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3628 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3629 bpf_prog_info_set_offset_u32(&info_linear->info, 3630 desc->count_offset, count); 3631 bpf_prog_info_set_offset_u32(&info_linear->info, 3632 desc->size_offset, size); 3633 bpf_prog_info_set_offset_u64(&info_linear->info, 3634 desc->array_offset, 3635 ptr_to_u64(ptr)); 3636 ptr += count * size; 3637 } 3638 3639 /* step 5: call syscall again to get required arrays */ 3640 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 3641 if (err) { 3642 pr_debug("can't get prog info: %s", strerror(errno)); 3643 free(info_linear); 3644 return ERR_PTR(-EFAULT); 3645 } 3646 3647 /* step 6: verify the data */ 3648 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3649 struct bpf_prog_info_array_desc *desc; 3650 __u32 v1, v2; 3651 3652 if ((arrays & (1UL << i)) == 0) 3653 continue; 3654 3655 desc = bpf_prog_info_array_desc + i; 3656 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3657 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3658 desc->count_offset); 3659 if (v1 != v2) 3660 pr_warning("%s: mismatch in element count\n", __func__); 3661 3662 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3663 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3664 desc->size_offset); 3665 if (v1 != v2) 3666 pr_warning("%s: mismatch in rec size\n", __func__); 3667 } 3668 3669 /* step 7: update info_len and data_len */ 3670 info_linear->info_len = sizeof(struct bpf_prog_info); 3671 info_linear->data_len = data_len; 3672 3673 return info_linear; 3674 } 3675 3676 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 3677 { 3678 int i; 3679 3680 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3681 struct bpf_prog_info_array_desc *desc; 3682 __u64 addr, offs; 3683 3684 if ((info_linear->arrays & (1UL << i)) == 0) 3685 continue; 3686 3687 desc = bpf_prog_info_array_desc + i; 3688 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 3689 desc->array_offset); 3690 offs = addr - ptr_to_u64(info_linear->data); 3691 bpf_prog_info_set_offset_u64(&info_linear->info, 3692 desc->array_offset, offs); 3693 } 3694 } 3695 3696 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 3697 { 3698 int i; 3699 3700 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3701 struct bpf_prog_info_array_desc *desc; 3702 __u64 addr, offs; 3703 3704 if ((info_linear->arrays & (1UL << i)) == 0) 3705 continue; 3706 3707 desc = bpf_prog_info_array_desc + i; 3708 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 3709 desc->array_offset); 3710 addr = offs + ptr_to_u64(info_linear->data); 3711 bpf_prog_info_set_offset_u64(&info_linear->info, 3712 desc->array_offset, addr); 3713 } 3714 } 3715