1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <string.h> 22 #include <unistd.h> 23 #include <fcntl.h> 24 #include <errno.h> 25 #include <asm/unistd.h> 26 #include <linux/err.h> 27 #include <linux/kernel.h> 28 #include <linux/bpf.h> 29 #include <linux/btf.h> 30 #include <linux/filter.h> 31 #include <linux/list.h> 32 #include <linux/limits.h> 33 #include <linux/perf_event.h> 34 #include <linux/ring_buffer.h> 35 #include <sys/stat.h> 36 #include <sys/types.h> 37 #include <sys/vfs.h> 38 #include <tools/libc_compat.h> 39 #include <libelf.h> 40 #include <gelf.h> 41 42 #include "libbpf.h" 43 #include "bpf.h" 44 #include "btf.h" 45 #include "str_error.h" 46 #include "libbpf_util.h" 47 48 #ifndef EM_BPF 49 #define EM_BPF 247 50 #endif 51 52 #ifndef BPF_FS_MAGIC 53 #define BPF_FS_MAGIC 0xcafe4a11 54 #endif 55 56 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 57 * compilation if user enables corresponding warning. Disable it explicitly. 58 */ 59 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 60 61 #define __printf(a, b) __attribute__((format(printf, a, b))) 62 63 static int __base_pr(enum libbpf_print_level level, const char *format, 64 va_list args) 65 { 66 if (level == LIBBPF_DEBUG) 67 return 0; 68 69 return vfprintf(stderr, format, args); 70 } 71 72 static libbpf_print_fn_t __libbpf_pr = __base_pr; 73 74 void libbpf_set_print(libbpf_print_fn_t fn) 75 { 76 __libbpf_pr = fn; 77 } 78 79 __printf(2, 3) 80 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 81 { 82 va_list args; 83 84 if (!__libbpf_pr) 85 return; 86 87 va_start(args, format); 88 __libbpf_pr(level, format, args); 89 va_end(args); 90 } 91 92 #define STRERR_BUFSIZE 128 93 94 #define CHECK_ERR(action, err, out) do { \ 95 err = action; \ 96 if (err) \ 97 goto out; \ 98 } while(0) 99 100 101 /* Copied from tools/perf/util/util.h */ 102 #ifndef zfree 103 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 104 #endif 105 106 #ifndef zclose 107 # define zclose(fd) ({ \ 108 int ___err = 0; \ 109 if ((fd) >= 0) \ 110 ___err = close((fd)); \ 111 fd = -1; \ 112 ___err; }) 113 #endif 114 115 #ifdef HAVE_LIBELF_MMAP_SUPPORT 116 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 117 #else 118 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 119 #endif 120 121 static inline __u64 ptr_to_u64(const void *ptr) 122 { 123 return (__u64) (unsigned long) ptr; 124 } 125 126 struct bpf_capabilities { 127 /* v4.14: kernel support for program & map names. */ 128 __u32 name:1; 129 /* v5.2: kernel support for global data sections. */ 130 __u32 global_data:1; 131 }; 132 133 /* 134 * bpf_prog should be a better name but it has been used in 135 * linux/filter.h. 136 */ 137 struct bpf_program { 138 /* Index in elf obj file, for relocation use. */ 139 int idx; 140 char *name; 141 int prog_ifindex; 142 char *section_name; 143 /* section_name with / replaced by _; makes recursive pinning 144 * in bpf_object__pin_programs easier 145 */ 146 char *pin_name; 147 struct bpf_insn *insns; 148 size_t insns_cnt, main_prog_cnt; 149 enum bpf_prog_type type; 150 151 struct reloc_desc { 152 enum { 153 RELO_LD64, 154 RELO_CALL, 155 RELO_DATA, 156 } type; 157 int insn_idx; 158 union { 159 int map_idx; 160 int text_off; 161 }; 162 } *reloc_desc; 163 int nr_reloc; 164 int log_level; 165 166 struct { 167 int nr; 168 int *fds; 169 } instances; 170 bpf_program_prep_t preprocessor; 171 172 struct bpf_object *obj; 173 void *priv; 174 bpf_program_clear_priv_t clear_priv; 175 176 enum bpf_attach_type expected_attach_type; 177 int btf_fd; 178 void *func_info; 179 __u32 func_info_rec_size; 180 __u32 func_info_cnt; 181 182 struct bpf_capabilities *caps; 183 184 void *line_info; 185 __u32 line_info_rec_size; 186 __u32 line_info_cnt; 187 }; 188 189 enum libbpf_map_type { 190 LIBBPF_MAP_UNSPEC, 191 LIBBPF_MAP_DATA, 192 LIBBPF_MAP_BSS, 193 LIBBPF_MAP_RODATA, 194 }; 195 196 static const char * const libbpf_type_to_btf_name[] = { 197 [LIBBPF_MAP_DATA] = ".data", 198 [LIBBPF_MAP_BSS] = ".bss", 199 [LIBBPF_MAP_RODATA] = ".rodata", 200 }; 201 202 struct bpf_map { 203 int fd; 204 char *name; 205 size_t offset; 206 int map_ifindex; 207 int inner_map_fd; 208 struct bpf_map_def def; 209 __u32 btf_key_type_id; 210 __u32 btf_value_type_id; 211 void *priv; 212 bpf_map_clear_priv_t clear_priv; 213 enum libbpf_map_type libbpf_type; 214 }; 215 216 struct bpf_secdata { 217 void *rodata; 218 void *data; 219 }; 220 221 static LIST_HEAD(bpf_objects_list); 222 223 struct bpf_object { 224 char name[BPF_OBJ_NAME_LEN]; 225 char license[64]; 226 __u32 kern_version; 227 228 struct bpf_program *programs; 229 size_t nr_programs; 230 struct bpf_map *maps; 231 size_t nr_maps; 232 struct bpf_secdata sections; 233 234 bool loaded; 235 bool has_pseudo_calls; 236 237 /* 238 * Information when doing elf related work. Only valid if fd 239 * is valid. 240 */ 241 struct { 242 int fd; 243 void *obj_buf; 244 size_t obj_buf_sz; 245 Elf *elf; 246 GElf_Ehdr ehdr; 247 Elf_Data *symbols; 248 Elf_Data *data; 249 Elf_Data *rodata; 250 Elf_Data *bss; 251 size_t strtabidx; 252 struct { 253 GElf_Shdr shdr; 254 Elf_Data *data; 255 } *reloc; 256 int nr_reloc; 257 int maps_shndx; 258 int text_shndx; 259 int data_shndx; 260 int rodata_shndx; 261 int bss_shndx; 262 } efile; 263 /* 264 * All loaded bpf_object is linked in a list, which is 265 * hidden to caller. bpf_objects__<func> handlers deal with 266 * all objects. 267 */ 268 struct list_head list; 269 270 struct btf *btf; 271 struct btf_ext *btf_ext; 272 273 void *priv; 274 bpf_object_clear_priv_t clear_priv; 275 276 struct bpf_capabilities caps; 277 278 char path[]; 279 }; 280 #define obj_elf_valid(o) ((o)->efile.elf) 281 282 void bpf_program__unload(struct bpf_program *prog) 283 { 284 int i; 285 286 if (!prog) 287 return; 288 289 /* 290 * If the object is opened but the program was never loaded, 291 * it is possible that prog->instances.nr == -1. 292 */ 293 if (prog->instances.nr > 0) { 294 for (i = 0; i < prog->instances.nr; i++) 295 zclose(prog->instances.fds[i]); 296 } else if (prog->instances.nr != -1) { 297 pr_warning("Internal error: instances.nr is %d\n", 298 prog->instances.nr); 299 } 300 301 prog->instances.nr = -1; 302 zfree(&prog->instances.fds); 303 304 zclose(prog->btf_fd); 305 zfree(&prog->func_info); 306 zfree(&prog->line_info); 307 } 308 309 static void bpf_program__exit(struct bpf_program *prog) 310 { 311 if (!prog) 312 return; 313 314 if (prog->clear_priv) 315 prog->clear_priv(prog, prog->priv); 316 317 prog->priv = NULL; 318 prog->clear_priv = NULL; 319 320 bpf_program__unload(prog); 321 zfree(&prog->name); 322 zfree(&prog->section_name); 323 zfree(&prog->pin_name); 324 zfree(&prog->insns); 325 zfree(&prog->reloc_desc); 326 327 prog->nr_reloc = 0; 328 prog->insns_cnt = 0; 329 prog->idx = -1; 330 } 331 332 static char *__bpf_program__pin_name(struct bpf_program *prog) 333 { 334 char *name, *p; 335 336 name = p = strdup(prog->section_name); 337 while ((p = strchr(p, '/'))) 338 *p = '_'; 339 340 return name; 341 } 342 343 static int 344 bpf_program__init(void *data, size_t size, char *section_name, int idx, 345 struct bpf_program *prog) 346 { 347 if (size < sizeof(struct bpf_insn)) { 348 pr_warning("corrupted section '%s'\n", section_name); 349 return -EINVAL; 350 } 351 352 memset(prog, 0, sizeof(*prog)); 353 354 prog->section_name = strdup(section_name); 355 if (!prog->section_name) { 356 pr_warning("failed to alloc name for prog under section(%d) %s\n", 357 idx, section_name); 358 goto errout; 359 } 360 361 prog->pin_name = __bpf_program__pin_name(prog); 362 if (!prog->pin_name) { 363 pr_warning("failed to alloc pin name for prog under section(%d) %s\n", 364 idx, section_name); 365 goto errout; 366 } 367 368 prog->insns = malloc(size); 369 if (!prog->insns) { 370 pr_warning("failed to alloc insns for prog under section %s\n", 371 section_name); 372 goto errout; 373 } 374 prog->insns_cnt = size / sizeof(struct bpf_insn); 375 memcpy(prog->insns, data, 376 prog->insns_cnt * sizeof(struct bpf_insn)); 377 prog->idx = idx; 378 prog->instances.fds = NULL; 379 prog->instances.nr = -1; 380 prog->type = BPF_PROG_TYPE_UNSPEC; 381 prog->btf_fd = -1; 382 383 return 0; 384 errout: 385 bpf_program__exit(prog); 386 return -ENOMEM; 387 } 388 389 static int 390 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 391 char *section_name, int idx) 392 { 393 struct bpf_program prog, *progs; 394 int nr_progs, err; 395 396 err = bpf_program__init(data, size, section_name, idx, &prog); 397 if (err) 398 return err; 399 400 prog.caps = &obj->caps; 401 progs = obj->programs; 402 nr_progs = obj->nr_programs; 403 404 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); 405 if (!progs) { 406 /* 407 * In this case the original obj->programs 408 * is still valid, so don't need special treat for 409 * bpf_close_object(). 410 */ 411 pr_warning("failed to alloc a new program under section '%s'\n", 412 section_name); 413 bpf_program__exit(&prog); 414 return -ENOMEM; 415 } 416 417 pr_debug("found program %s\n", prog.section_name); 418 obj->programs = progs; 419 obj->nr_programs = nr_progs + 1; 420 prog.obj = obj; 421 progs[nr_progs] = prog; 422 return 0; 423 } 424 425 static int 426 bpf_object__init_prog_names(struct bpf_object *obj) 427 { 428 Elf_Data *symbols = obj->efile.symbols; 429 struct bpf_program *prog; 430 size_t pi, si; 431 432 for (pi = 0; pi < obj->nr_programs; pi++) { 433 const char *name = NULL; 434 435 prog = &obj->programs[pi]; 436 437 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 438 si++) { 439 GElf_Sym sym; 440 441 if (!gelf_getsym(symbols, si, &sym)) 442 continue; 443 if (sym.st_shndx != prog->idx) 444 continue; 445 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 446 continue; 447 448 name = elf_strptr(obj->efile.elf, 449 obj->efile.strtabidx, 450 sym.st_name); 451 if (!name) { 452 pr_warning("failed to get sym name string for prog %s\n", 453 prog->section_name); 454 return -LIBBPF_ERRNO__LIBELF; 455 } 456 } 457 458 if (!name && prog->idx == obj->efile.text_shndx) 459 name = ".text"; 460 461 if (!name) { 462 pr_warning("failed to find sym for prog %s\n", 463 prog->section_name); 464 return -EINVAL; 465 } 466 467 prog->name = strdup(name); 468 if (!prog->name) { 469 pr_warning("failed to allocate memory for prog sym %s\n", 470 name); 471 return -ENOMEM; 472 } 473 } 474 475 return 0; 476 } 477 478 static struct bpf_object *bpf_object__new(const char *path, 479 void *obj_buf, 480 size_t obj_buf_sz) 481 { 482 struct bpf_object *obj; 483 char *end; 484 485 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 486 if (!obj) { 487 pr_warning("alloc memory failed for %s\n", path); 488 return ERR_PTR(-ENOMEM); 489 } 490 491 strcpy(obj->path, path); 492 /* Using basename() GNU version which doesn't modify arg. */ 493 strncpy(obj->name, basename((void *)path), 494 sizeof(obj->name) - 1); 495 end = strchr(obj->name, '.'); 496 if (end) 497 *end = 0; 498 499 obj->efile.fd = -1; 500 /* 501 * Caller of this function should also calls 502 * bpf_object__elf_finish() after data collection to return 503 * obj_buf to user. If not, we should duplicate the buffer to 504 * avoid user freeing them before elf finish. 505 */ 506 obj->efile.obj_buf = obj_buf; 507 obj->efile.obj_buf_sz = obj_buf_sz; 508 obj->efile.maps_shndx = -1; 509 obj->efile.data_shndx = -1; 510 obj->efile.rodata_shndx = -1; 511 obj->efile.bss_shndx = -1; 512 513 obj->loaded = false; 514 515 INIT_LIST_HEAD(&obj->list); 516 list_add(&obj->list, &bpf_objects_list); 517 return obj; 518 } 519 520 static void bpf_object__elf_finish(struct bpf_object *obj) 521 { 522 if (!obj_elf_valid(obj)) 523 return; 524 525 if (obj->efile.elf) { 526 elf_end(obj->efile.elf); 527 obj->efile.elf = NULL; 528 } 529 obj->efile.symbols = NULL; 530 obj->efile.data = NULL; 531 obj->efile.rodata = NULL; 532 obj->efile.bss = NULL; 533 534 zfree(&obj->efile.reloc); 535 obj->efile.nr_reloc = 0; 536 zclose(obj->efile.fd); 537 obj->efile.obj_buf = NULL; 538 obj->efile.obj_buf_sz = 0; 539 } 540 541 static int bpf_object__elf_init(struct bpf_object *obj) 542 { 543 int err = 0; 544 GElf_Ehdr *ep; 545 546 if (obj_elf_valid(obj)) { 547 pr_warning("elf init: internal error\n"); 548 return -LIBBPF_ERRNO__LIBELF; 549 } 550 551 if (obj->efile.obj_buf_sz > 0) { 552 /* 553 * obj_buf should have been validated by 554 * bpf_object__open_buffer(). 555 */ 556 obj->efile.elf = elf_memory(obj->efile.obj_buf, 557 obj->efile.obj_buf_sz); 558 } else { 559 obj->efile.fd = open(obj->path, O_RDONLY); 560 if (obj->efile.fd < 0) { 561 char errmsg[STRERR_BUFSIZE]; 562 char *cp = libbpf_strerror_r(errno, errmsg, 563 sizeof(errmsg)); 564 565 pr_warning("failed to open %s: %s\n", obj->path, cp); 566 return -errno; 567 } 568 569 obj->efile.elf = elf_begin(obj->efile.fd, 570 LIBBPF_ELF_C_READ_MMAP, 571 NULL); 572 } 573 574 if (!obj->efile.elf) { 575 pr_warning("failed to open %s as ELF file\n", 576 obj->path); 577 err = -LIBBPF_ERRNO__LIBELF; 578 goto errout; 579 } 580 581 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 582 pr_warning("failed to get EHDR from %s\n", 583 obj->path); 584 err = -LIBBPF_ERRNO__FORMAT; 585 goto errout; 586 } 587 ep = &obj->efile.ehdr; 588 589 /* Old LLVM set e_machine to EM_NONE */ 590 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { 591 pr_warning("%s is not an eBPF object file\n", 592 obj->path); 593 err = -LIBBPF_ERRNO__FORMAT; 594 goto errout; 595 } 596 597 return 0; 598 errout: 599 bpf_object__elf_finish(obj); 600 return err; 601 } 602 603 static int 604 bpf_object__check_endianness(struct bpf_object *obj) 605 { 606 static unsigned int const endian = 1; 607 608 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 609 case ELFDATA2LSB: 610 /* We are big endian, BPF obj is little endian. */ 611 if (*(unsigned char const *)&endian != 1) 612 goto mismatch; 613 break; 614 615 case ELFDATA2MSB: 616 /* We are little endian, BPF obj is big endian. */ 617 if (*(unsigned char const *)&endian != 0) 618 goto mismatch; 619 break; 620 default: 621 return -LIBBPF_ERRNO__ENDIAN; 622 } 623 624 return 0; 625 626 mismatch: 627 pr_warning("Error: endianness mismatch.\n"); 628 return -LIBBPF_ERRNO__ENDIAN; 629 } 630 631 static int 632 bpf_object__init_license(struct bpf_object *obj, 633 void *data, size_t size) 634 { 635 memcpy(obj->license, data, 636 min(size, sizeof(obj->license) - 1)); 637 pr_debug("license of %s is %s\n", obj->path, obj->license); 638 return 0; 639 } 640 641 static int 642 bpf_object__init_kversion(struct bpf_object *obj, 643 void *data, size_t size) 644 { 645 __u32 kver; 646 647 if (size != sizeof(kver)) { 648 pr_warning("invalid kver section in %s\n", obj->path); 649 return -LIBBPF_ERRNO__FORMAT; 650 } 651 memcpy(&kver, data, sizeof(kver)); 652 obj->kern_version = kver; 653 pr_debug("kernel version of %s is %x\n", obj->path, 654 obj->kern_version); 655 return 0; 656 } 657 658 static int compare_bpf_map(const void *_a, const void *_b) 659 { 660 const struct bpf_map *a = _a; 661 const struct bpf_map *b = _b; 662 663 return a->offset - b->offset; 664 } 665 666 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 667 { 668 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 669 type == BPF_MAP_TYPE_HASH_OF_MAPS) 670 return true; 671 return false; 672 } 673 674 static int bpf_object_search_section_size(const struct bpf_object *obj, 675 const char *name, size_t *d_size) 676 { 677 const GElf_Ehdr *ep = &obj->efile.ehdr; 678 Elf *elf = obj->efile.elf; 679 Elf_Scn *scn = NULL; 680 int idx = 0; 681 682 while ((scn = elf_nextscn(elf, scn)) != NULL) { 683 const char *sec_name; 684 Elf_Data *data; 685 GElf_Shdr sh; 686 687 idx++; 688 if (gelf_getshdr(scn, &sh) != &sh) { 689 pr_warning("failed to get section(%d) header from %s\n", 690 idx, obj->path); 691 return -EIO; 692 } 693 694 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 695 if (!sec_name) { 696 pr_warning("failed to get section(%d) name from %s\n", 697 idx, obj->path); 698 return -EIO; 699 } 700 701 if (strcmp(name, sec_name)) 702 continue; 703 704 data = elf_getdata(scn, 0); 705 if (!data) { 706 pr_warning("failed to get section(%d) data from %s(%s)\n", 707 idx, name, obj->path); 708 return -EIO; 709 } 710 711 *d_size = data->d_size; 712 return 0; 713 } 714 715 return -ENOENT; 716 } 717 718 int bpf_object__section_size(const struct bpf_object *obj, const char *name, 719 __u32 *size) 720 { 721 int ret = -ENOENT; 722 size_t d_size; 723 724 *size = 0; 725 if (!name) { 726 return -EINVAL; 727 } else if (!strcmp(name, ".data")) { 728 if (obj->efile.data) 729 *size = obj->efile.data->d_size; 730 } else if (!strcmp(name, ".bss")) { 731 if (obj->efile.bss) 732 *size = obj->efile.bss->d_size; 733 } else if (!strcmp(name, ".rodata")) { 734 if (obj->efile.rodata) 735 *size = obj->efile.rodata->d_size; 736 } else { 737 ret = bpf_object_search_section_size(obj, name, &d_size); 738 if (!ret) 739 *size = d_size; 740 } 741 742 return *size ? 0 : ret; 743 } 744 745 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, 746 __u32 *off) 747 { 748 Elf_Data *symbols = obj->efile.symbols; 749 const char *sname; 750 size_t si; 751 752 if (!name || !off) 753 return -EINVAL; 754 755 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { 756 GElf_Sym sym; 757 758 if (!gelf_getsym(symbols, si, &sym)) 759 continue; 760 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || 761 GELF_ST_TYPE(sym.st_info) != STT_OBJECT) 762 continue; 763 764 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 765 sym.st_name); 766 if (!sname) { 767 pr_warning("failed to get sym name string for var %s\n", 768 name); 769 return -EIO; 770 } 771 if (strcmp(name, sname) == 0) { 772 *off = sym.st_value; 773 return 0; 774 } 775 } 776 777 return -ENOENT; 778 } 779 780 static bool bpf_object__has_maps(const struct bpf_object *obj) 781 { 782 return obj->efile.maps_shndx >= 0 || 783 obj->efile.data_shndx >= 0 || 784 obj->efile.rodata_shndx >= 0 || 785 obj->efile.bss_shndx >= 0; 786 } 787 788 static int 789 bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map, 790 enum libbpf_map_type type, Elf_Data *data, 791 void **data_buff) 792 { 793 struct bpf_map_def *def = &map->def; 794 char map_name[BPF_OBJ_NAME_LEN]; 795 796 map->libbpf_type = type; 797 map->offset = ~(typeof(map->offset))0; 798 snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, 799 libbpf_type_to_btf_name[type]); 800 map->name = strdup(map_name); 801 if (!map->name) { 802 pr_warning("failed to alloc map name\n"); 803 return -ENOMEM; 804 } 805 806 def->type = BPF_MAP_TYPE_ARRAY; 807 def->key_size = sizeof(int); 808 def->value_size = data->d_size; 809 def->max_entries = 1; 810 def->map_flags = type == LIBBPF_MAP_RODATA ? 811 BPF_F_RDONLY_PROG : 0; 812 if (data_buff) { 813 *data_buff = malloc(data->d_size); 814 if (!*data_buff) { 815 zfree(&map->name); 816 pr_warning("failed to alloc map content buffer\n"); 817 return -ENOMEM; 818 } 819 memcpy(*data_buff, data->d_buf, data->d_size); 820 } 821 822 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 823 return 0; 824 } 825 826 static int 827 bpf_object__init_maps(struct bpf_object *obj, int flags) 828 { 829 int i, map_idx, map_def_sz = 0, nr_syms, nr_maps = 0, nr_maps_glob = 0; 830 bool strict = !(flags & MAPS_RELAX_COMPAT); 831 Elf_Data *symbols = obj->efile.symbols; 832 Elf_Data *data = NULL; 833 int ret = 0; 834 835 if (!symbols) 836 return -EINVAL; 837 nr_syms = symbols->d_size / sizeof(GElf_Sym); 838 839 if (obj->efile.maps_shndx >= 0) { 840 Elf_Scn *scn = elf_getscn(obj->efile.elf, 841 obj->efile.maps_shndx); 842 843 if (scn) 844 data = elf_getdata(scn, NULL); 845 if (!scn || !data) { 846 pr_warning("failed to get Elf_Data from map section %d\n", 847 obj->efile.maps_shndx); 848 return -EINVAL; 849 } 850 } 851 852 /* 853 * Count number of maps. Each map has a name. 854 * Array of maps is not supported: only the first element is 855 * considered. 856 * 857 * TODO: Detect array of map and report error. 858 */ 859 if (obj->caps.global_data) { 860 if (obj->efile.data_shndx >= 0) 861 nr_maps_glob++; 862 if (obj->efile.rodata_shndx >= 0) 863 nr_maps_glob++; 864 if (obj->efile.bss_shndx >= 0) 865 nr_maps_glob++; 866 } 867 868 for (i = 0; data && i < nr_syms; i++) { 869 GElf_Sym sym; 870 871 if (!gelf_getsym(symbols, i, &sym)) 872 continue; 873 if (sym.st_shndx != obj->efile.maps_shndx) 874 continue; 875 nr_maps++; 876 } 877 878 if (!nr_maps && !nr_maps_glob) 879 return 0; 880 881 /* Assume equally sized map definitions */ 882 if (data) { 883 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 884 nr_maps, data->d_size); 885 886 map_def_sz = data->d_size / nr_maps; 887 if (!data->d_size || (data->d_size % nr_maps) != 0) { 888 pr_warning("unable to determine map definition size " 889 "section %s, %d maps in %zd bytes\n", 890 obj->path, nr_maps, data->d_size); 891 return -EINVAL; 892 } 893 } 894 895 nr_maps += nr_maps_glob; 896 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 897 if (!obj->maps) { 898 pr_warning("alloc maps for object failed\n"); 899 return -ENOMEM; 900 } 901 obj->nr_maps = nr_maps; 902 903 for (i = 0; i < nr_maps; i++) { 904 /* 905 * fill all fd with -1 so won't close incorrect 906 * fd (fd=0 is stdin) when failure (zclose won't close 907 * negative fd)). 908 */ 909 obj->maps[i].fd = -1; 910 obj->maps[i].inner_map_fd = -1; 911 } 912 913 /* 914 * Fill obj->maps using data in "maps" section. 915 */ 916 for (i = 0, map_idx = 0; data && i < nr_syms; i++) { 917 GElf_Sym sym; 918 const char *map_name; 919 struct bpf_map_def *def; 920 921 if (!gelf_getsym(symbols, i, &sym)) 922 continue; 923 if (sym.st_shndx != obj->efile.maps_shndx) 924 continue; 925 926 map_name = elf_strptr(obj->efile.elf, 927 obj->efile.strtabidx, 928 sym.st_name); 929 930 obj->maps[map_idx].libbpf_type = LIBBPF_MAP_UNSPEC; 931 obj->maps[map_idx].offset = sym.st_value; 932 if (sym.st_value + map_def_sz > data->d_size) { 933 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 934 obj->path, map_name); 935 return -EINVAL; 936 } 937 938 obj->maps[map_idx].name = strdup(map_name); 939 if (!obj->maps[map_idx].name) { 940 pr_warning("failed to alloc map name\n"); 941 return -ENOMEM; 942 } 943 pr_debug("map %d is \"%s\"\n", map_idx, 944 obj->maps[map_idx].name); 945 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 946 /* 947 * If the definition of the map in the object file fits in 948 * bpf_map_def, copy it. Any extra fields in our version 949 * of bpf_map_def will default to zero as a result of the 950 * calloc above. 951 */ 952 if (map_def_sz <= sizeof(struct bpf_map_def)) { 953 memcpy(&obj->maps[map_idx].def, def, map_def_sz); 954 } else { 955 /* 956 * Here the map structure being read is bigger than what 957 * we expect, truncate if the excess bits are all zero. 958 * If they are not zero, reject this map as 959 * incompatible. 960 */ 961 char *b; 962 for (b = ((char *)def) + sizeof(struct bpf_map_def); 963 b < ((char *)def) + map_def_sz; b++) { 964 if (*b != 0) { 965 pr_warning("maps section in %s: \"%s\" " 966 "has unrecognized, non-zero " 967 "options\n", 968 obj->path, map_name); 969 if (strict) 970 return -EINVAL; 971 } 972 } 973 memcpy(&obj->maps[map_idx].def, def, 974 sizeof(struct bpf_map_def)); 975 } 976 map_idx++; 977 } 978 979 if (!obj->caps.global_data) 980 goto finalize; 981 982 /* 983 * Populate rest of obj->maps with libbpf internal maps. 984 */ 985 if (obj->efile.data_shndx >= 0) 986 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 987 LIBBPF_MAP_DATA, 988 obj->efile.data, 989 &obj->sections.data); 990 if (!ret && obj->efile.rodata_shndx >= 0) 991 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 992 LIBBPF_MAP_RODATA, 993 obj->efile.rodata, 994 &obj->sections.rodata); 995 if (!ret && obj->efile.bss_shndx >= 0) 996 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], 997 LIBBPF_MAP_BSS, 998 obj->efile.bss, NULL); 999 finalize: 1000 if (!ret) 1001 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), 1002 compare_bpf_map); 1003 return ret; 1004 } 1005 1006 static bool section_have_execinstr(struct bpf_object *obj, int idx) 1007 { 1008 Elf_Scn *scn; 1009 GElf_Shdr sh; 1010 1011 scn = elf_getscn(obj->efile.elf, idx); 1012 if (!scn) 1013 return false; 1014 1015 if (gelf_getshdr(scn, &sh) != &sh) 1016 return false; 1017 1018 if (sh.sh_flags & SHF_EXECINSTR) 1019 return true; 1020 1021 return false; 1022 } 1023 1024 static int bpf_object__elf_collect(struct bpf_object *obj, int flags) 1025 { 1026 Elf *elf = obj->efile.elf; 1027 GElf_Ehdr *ep = &obj->efile.ehdr; 1028 Elf_Data *btf_ext_data = NULL; 1029 Elf_Data *btf_data = NULL; 1030 Elf_Scn *scn = NULL; 1031 int idx = 0, err = 0; 1032 1033 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1034 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 1035 pr_warning("failed to get e_shstrndx from %s\n", 1036 obj->path); 1037 return -LIBBPF_ERRNO__FORMAT; 1038 } 1039 1040 while ((scn = elf_nextscn(elf, scn)) != NULL) { 1041 char *name; 1042 GElf_Shdr sh; 1043 Elf_Data *data; 1044 1045 idx++; 1046 if (gelf_getshdr(scn, &sh) != &sh) { 1047 pr_warning("failed to get section(%d) header from %s\n", 1048 idx, obj->path); 1049 err = -LIBBPF_ERRNO__FORMAT; 1050 goto out; 1051 } 1052 1053 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 1054 if (!name) { 1055 pr_warning("failed to get section(%d) name from %s\n", 1056 idx, obj->path); 1057 err = -LIBBPF_ERRNO__FORMAT; 1058 goto out; 1059 } 1060 1061 data = elf_getdata(scn, 0); 1062 if (!data) { 1063 pr_warning("failed to get section(%d) data from %s(%s)\n", 1064 idx, name, obj->path); 1065 err = -LIBBPF_ERRNO__FORMAT; 1066 goto out; 1067 } 1068 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 1069 idx, name, (unsigned long)data->d_size, 1070 (int)sh.sh_link, (unsigned long)sh.sh_flags, 1071 (int)sh.sh_type); 1072 1073 if (strcmp(name, "license") == 0) { 1074 err = bpf_object__init_license(obj, 1075 data->d_buf, 1076 data->d_size); 1077 } else if (strcmp(name, "version") == 0) { 1078 err = bpf_object__init_kversion(obj, 1079 data->d_buf, 1080 data->d_size); 1081 } else if (strcmp(name, "maps") == 0) { 1082 obj->efile.maps_shndx = idx; 1083 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 1084 btf_data = data; 1085 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 1086 btf_ext_data = data; 1087 } else if (sh.sh_type == SHT_SYMTAB) { 1088 if (obj->efile.symbols) { 1089 pr_warning("bpf: multiple SYMTAB in %s\n", 1090 obj->path); 1091 err = -LIBBPF_ERRNO__FORMAT; 1092 } else { 1093 obj->efile.symbols = data; 1094 obj->efile.strtabidx = sh.sh_link; 1095 } 1096 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { 1097 if (sh.sh_flags & SHF_EXECINSTR) { 1098 if (strcmp(name, ".text") == 0) 1099 obj->efile.text_shndx = idx; 1100 err = bpf_object__add_program(obj, data->d_buf, 1101 data->d_size, name, idx); 1102 if (err) { 1103 char errmsg[STRERR_BUFSIZE]; 1104 char *cp = libbpf_strerror_r(-err, errmsg, 1105 sizeof(errmsg)); 1106 1107 pr_warning("failed to alloc program %s (%s): %s", 1108 name, obj->path, cp); 1109 } 1110 } else if (strcmp(name, ".data") == 0) { 1111 obj->efile.data = data; 1112 obj->efile.data_shndx = idx; 1113 } else if (strcmp(name, ".rodata") == 0) { 1114 obj->efile.rodata = data; 1115 obj->efile.rodata_shndx = idx; 1116 } else { 1117 pr_debug("skip section(%d) %s\n", idx, name); 1118 } 1119 } else if (sh.sh_type == SHT_REL) { 1120 void *reloc = obj->efile.reloc; 1121 int nr_reloc = obj->efile.nr_reloc + 1; 1122 int sec = sh.sh_info; /* points to other section */ 1123 1124 /* Only do relo for section with exec instructions */ 1125 if (!section_have_execinstr(obj, sec)) { 1126 pr_debug("skip relo %s(%d) for section(%d)\n", 1127 name, idx, sec); 1128 continue; 1129 } 1130 1131 reloc = reallocarray(reloc, nr_reloc, 1132 sizeof(*obj->efile.reloc)); 1133 if (!reloc) { 1134 pr_warning("realloc failed\n"); 1135 err = -ENOMEM; 1136 } else { 1137 int n = nr_reloc - 1; 1138 1139 obj->efile.reloc = reloc; 1140 obj->efile.nr_reloc = nr_reloc; 1141 1142 obj->efile.reloc[n].shdr = sh; 1143 obj->efile.reloc[n].data = data; 1144 } 1145 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { 1146 obj->efile.bss = data; 1147 obj->efile.bss_shndx = idx; 1148 } else { 1149 pr_debug("skip section(%d) %s\n", idx, name); 1150 } 1151 if (err) 1152 goto out; 1153 } 1154 1155 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 1156 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 1157 return LIBBPF_ERRNO__FORMAT; 1158 } 1159 if (btf_data) { 1160 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 1161 if (IS_ERR(obj->btf)) { 1162 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1163 BTF_ELF_SEC, PTR_ERR(obj->btf)); 1164 obj->btf = NULL; 1165 } else { 1166 err = btf__finalize_data(obj, obj->btf); 1167 if (!err) 1168 err = btf__load(obj->btf); 1169 if (err) { 1170 pr_warning("Error finalizing and loading %s into kernel: %d. Ignored and continue.\n", 1171 BTF_ELF_SEC, err); 1172 btf__free(obj->btf); 1173 obj->btf = NULL; 1174 err = 0; 1175 } 1176 } 1177 } 1178 if (btf_ext_data) { 1179 if (!obj->btf) { 1180 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 1181 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 1182 } else { 1183 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, 1184 btf_ext_data->d_size); 1185 if (IS_ERR(obj->btf_ext)) { 1186 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", 1187 BTF_EXT_ELF_SEC, 1188 PTR_ERR(obj->btf_ext)); 1189 obj->btf_ext = NULL; 1190 } 1191 } 1192 } 1193 if (bpf_object__has_maps(obj)) { 1194 err = bpf_object__init_maps(obj, flags); 1195 if (err) 1196 goto out; 1197 } 1198 err = bpf_object__init_prog_names(obj); 1199 out: 1200 return err; 1201 } 1202 1203 static struct bpf_program * 1204 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 1205 { 1206 struct bpf_program *prog; 1207 size_t i; 1208 1209 for (i = 0; i < obj->nr_programs; i++) { 1210 prog = &obj->programs[i]; 1211 if (prog->idx == idx) 1212 return prog; 1213 } 1214 return NULL; 1215 } 1216 1217 struct bpf_program * 1218 bpf_object__find_program_by_title(struct bpf_object *obj, const char *title) 1219 { 1220 struct bpf_program *pos; 1221 1222 bpf_object__for_each_program(pos, obj) { 1223 if (pos->section_name && !strcmp(pos->section_name, title)) 1224 return pos; 1225 } 1226 return NULL; 1227 } 1228 1229 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 1230 int shndx) 1231 { 1232 return shndx == obj->efile.data_shndx || 1233 shndx == obj->efile.bss_shndx || 1234 shndx == obj->efile.rodata_shndx; 1235 } 1236 1237 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 1238 int shndx) 1239 { 1240 return shndx == obj->efile.maps_shndx; 1241 } 1242 1243 static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, 1244 int shndx) 1245 { 1246 return shndx == obj->efile.text_shndx || 1247 bpf_object__shndx_is_maps(obj, shndx) || 1248 bpf_object__shndx_is_data(obj, shndx); 1249 } 1250 1251 static enum libbpf_map_type 1252 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 1253 { 1254 if (shndx == obj->efile.data_shndx) 1255 return LIBBPF_MAP_DATA; 1256 else if (shndx == obj->efile.bss_shndx) 1257 return LIBBPF_MAP_BSS; 1258 else if (shndx == obj->efile.rodata_shndx) 1259 return LIBBPF_MAP_RODATA; 1260 else 1261 return LIBBPF_MAP_UNSPEC; 1262 } 1263 1264 static int 1265 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 1266 Elf_Data *data, struct bpf_object *obj) 1267 { 1268 Elf_Data *symbols = obj->efile.symbols; 1269 struct bpf_map *maps = obj->maps; 1270 size_t nr_maps = obj->nr_maps; 1271 int i, nrels; 1272 1273 pr_debug("collecting relocating info for: '%s'\n", 1274 prog->section_name); 1275 nrels = shdr->sh_size / shdr->sh_entsize; 1276 1277 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 1278 if (!prog->reloc_desc) { 1279 pr_warning("failed to alloc memory in relocation\n"); 1280 return -ENOMEM; 1281 } 1282 prog->nr_reloc = nrels; 1283 1284 for (i = 0; i < nrels; i++) { 1285 GElf_Sym sym; 1286 GElf_Rel rel; 1287 unsigned int insn_idx; 1288 unsigned int shdr_idx; 1289 struct bpf_insn *insns = prog->insns; 1290 enum libbpf_map_type type; 1291 const char *name; 1292 size_t map_idx; 1293 1294 if (!gelf_getrel(data, i, &rel)) { 1295 pr_warning("relocation: failed to get %d reloc\n", i); 1296 return -LIBBPF_ERRNO__FORMAT; 1297 } 1298 1299 if (!gelf_getsym(symbols, 1300 GELF_R_SYM(rel.r_info), 1301 &sym)) { 1302 pr_warning("relocation: symbol %"PRIx64" not found\n", 1303 GELF_R_SYM(rel.r_info)); 1304 return -LIBBPF_ERRNO__FORMAT; 1305 } 1306 1307 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, 1308 sym.st_name) ? : "<?>"; 1309 1310 pr_debug("relo for %lld value %lld name %d (\'%s\')\n", 1311 (long long) (rel.r_info >> 32), 1312 (long long) sym.st_value, sym.st_name, name); 1313 1314 shdr_idx = sym.st_shndx; 1315 if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { 1316 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", 1317 prog->section_name, shdr_idx); 1318 return -LIBBPF_ERRNO__RELOC; 1319 } 1320 1321 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 1322 pr_debug("relocation: insn_idx=%u\n", insn_idx); 1323 1324 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 1325 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 1326 pr_warning("incorrect bpf_call opcode\n"); 1327 return -LIBBPF_ERRNO__RELOC; 1328 } 1329 prog->reloc_desc[i].type = RELO_CALL; 1330 prog->reloc_desc[i].insn_idx = insn_idx; 1331 prog->reloc_desc[i].text_off = sym.st_value; 1332 obj->has_pseudo_calls = true; 1333 continue; 1334 } 1335 1336 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 1337 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 1338 insn_idx, insns[insn_idx].code); 1339 return -LIBBPF_ERRNO__RELOC; 1340 } 1341 1342 if (bpf_object__shndx_is_maps(obj, shdr_idx) || 1343 bpf_object__shndx_is_data(obj, shdr_idx)) { 1344 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 1345 if (type != LIBBPF_MAP_UNSPEC) { 1346 if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { 1347 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", 1348 name, insn_idx, insns[insn_idx].code); 1349 return -LIBBPF_ERRNO__RELOC; 1350 } 1351 if (!obj->caps.global_data) { 1352 pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", 1353 name, insn_idx); 1354 return -LIBBPF_ERRNO__RELOC; 1355 } 1356 } 1357 1358 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 1359 if (maps[map_idx].libbpf_type != type) 1360 continue; 1361 if (type != LIBBPF_MAP_UNSPEC || 1362 (type == LIBBPF_MAP_UNSPEC && 1363 maps[map_idx].offset == sym.st_value)) { 1364 pr_debug("relocation: find map %zd (%s) for insn %u\n", 1365 map_idx, maps[map_idx].name, insn_idx); 1366 break; 1367 } 1368 } 1369 1370 if (map_idx >= nr_maps) { 1371 pr_warning("bpf relocation: map_idx %d large than %d\n", 1372 (int)map_idx, (int)nr_maps - 1); 1373 return -LIBBPF_ERRNO__RELOC; 1374 } 1375 1376 prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? 1377 RELO_DATA : RELO_LD64; 1378 prog->reloc_desc[i].insn_idx = insn_idx; 1379 prog->reloc_desc[i].map_idx = map_idx; 1380 } 1381 } 1382 return 0; 1383 } 1384 1385 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) 1386 { 1387 struct bpf_map_def *def = &map->def; 1388 __u32 key_type_id = 0, value_type_id = 0; 1389 int ret; 1390 1391 if (!bpf_map__is_internal(map)) { 1392 ret = btf__get_map_kv_tids(btf, map->name, def->key_size, 1393 def->value_size, &key_type_id, 1394 &value_type_id); 1395 } else { 1396 /* 1397 * LLVM annotates global data differently in BTF, that is, 1398 * only as '.data', '.bss' or '.rodata'. 1399 */ 1400 ret = btf__find_by_name(btf, 1401 libbpf_type_to_btf_name[map->libbpf_type]); 1402 } 1403 if (ret < 0) 1404 return ret; 1405 1406 map->btf_key_type_id = key_type_id; 1407 map->btf_value_type_id = bpf_map__is_internal(map) ? 1408 ret : value_type_id; 1409 return 0; 1410 } 1411 1412 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 1413 { 1414 struct bpf_map_info info = {}; 1415 __u32 len = sizeof(info); 1416 int new_fd, err; 1417 char *new_name; 1418 1419 err = bpf_obj_get_info_by_fd(fd, &info, &len); 1420 if (err) 1421 return err; 1422 1423 new_name = strdup(info.name); 1424 if (!new_name) 1425 return -errno; 1426 1427 new_fd = open("/", O_RDONLY | O_CLOEXEC); 1428 if (new_fd < 0) 1429 goto err_free_new_name; 1430 1431 new_fd = dup3(fd, new_fd, O_CLOEXEC); 1432 if (new_fd < 0) 1433 goto err_close_new_fd; 1434 1435 err = zclose(map->fd); 1436 if (err) 1437 goto err_close_new_fd; 1438 free(map->name); 1439 1440 map->fd = new_fd; 1441 map->name = new_name; 1442 map->def.type = info.type; 1443 map->def.key_size = info.key_size; 1444 map->def.value_size = info.value_size; 1445 map->def.max_entries = info.max_entries; 1446 map->def.map_flags = info.map_flags; 1447 map->btf_key_type_id = info.btf_key_type_id; 1448 map->btf_value_type_id = info.btf_value_type_id; 1449 1450 return 0; 1451 1452 err_close_new_fd: 1453 close(new_fd); 1454 err_free_new_name: 1455 free(new_name); 1456 return -errno; 1457 } 1458 1459 int bpf_map__resize(struct bpf_map *map, __u32 max_entries) 1460 { 1461 if (!map || !max_entries) 1462 return -EINVAL; 1463 1464 /* If map already created, its attributes can't be changed. */ 1465 if (map->fd >= 0) 1466 return -EBUSY; 1467 1468 map->def.max_entries = max_entries; 1469 1470 return 0; 1471 } 1472 1473 static int 1474 bpf_object__probe_name(struct bpf_object *obj) 1475 { 1476 struct bpf_load_program_attr attr; 1477 char *cp, errmsg[STRERR_BUFSIZE]; 1478 struct bpf_insn insns[] = { 1479 BPF_MOV64_IMM(BPF_REG_0, 0), 1480 BPF_EXIT_INSN(), 1481 }; 1482 int ret; 1483 1484 /* make sure basic loading works */ 1485 1486 memset(&attr, 0, sizeof(attr)); 1487 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1488 attr.insns = insns; 1489 attr.insns_cnt = ARRAY_SIZE(insns); 1490 attr.license = "GPL"; 1491 1492 ret = bpf_load_program_xattr(&attr, NULL, 0); 1493 if (ret < 0) { 1494 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1495 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", 1496 __func__, cp, errno); 1497 return -errno; 1498 } 1499 close(ret); 1500 1501 /* now try the same program, but with the name */ 1502 1503 attr.name = "test"; 1504 ret = bpf_load_program_xattr(&attr, NULL, 0); 1505 if (ret >= 0) { 1506 obj->caps.name = 1; 1507 close(ret); 1508 } 1509 1510 return 0; 1511 } 1512 1513 static int 1514 bpf_object__probe_global_data(struct bpf_object *obj) 1515 { 1516 struct bpf_load_program_attr prg_attr; 1517 struct bpf_create_map_attr map_attr; 1518 char *cp, errmsg[STRERR_BUFSIZE]; 1519 struct bpf_insn insns[] = { 1520 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 1521 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 1522 BPF_MOV64_IMM(BPF_REG_0, 0), 1523 BPF_EXIT_INSN(), 1524 }; 1525 int ret, map; 1526 1527 memset(&map_attr, 0, sizeof(map_attr)); 1528 map_attr.map_type = BPF_MAP_TYPE_ARRAY; 1529 map_attr.key_size = sizeof(int); 1530 map_attr.value_size = 32; 1531 map_attr.max_entries = 1; 1532 1533 map = bpf_create_map_xattr(&map_attr); 1534 if (map < 0) { 1535 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1536 pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", 1537 __func__, cp, errno); 1538 return -errno; 1539 } 1540 1541 insns[0].imm = map; 1542 1543 memset(&prg_attr, 0, sizeof(prg_attr)); 1544 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 1545 prg_attr.insns = insns; 1546 prg_attr.insns_cnt = ARRAY_SIZE(insns); 1547 prg_attr.license = "GPL"; 1548 1549 ret = bpf_load_program_xattr(&prg_attr, NULL, 0); 1550 if (ret >= 0) { 1551 obj->caps.global_data = 1; 1552 close(ret); 1553 } 1554 1555 close(map); 1556 return 0; 1557 } 1558 1559 static int 1560 bpf_object__probe_caps(struct bpf_object *obj) 1561 { 1562 int (*probe_fn[])(struct bpf_object *obj) = { 1563 bpf_object__probe_name, 1564 bpf_object__probe_global_data, 1565 }; 1566 int i, ret; 1567 1568 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { 1569 ret = probe_fn[i](obj); 1570 if (ret < 0) 1571 return ret; 1572 } 1573 1574 return 0; 1575 } 1576 1577 static int 1578 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 1579 { 1580 char *cp, errmsg[STRERR_BUFSIZE]; 1581 int err, zero = 0; 1582 __u8 *data; 1583 1584 /* Nothing to do here since kernel already zero-initializes .bss map. */ 1585 if (map->libbpf_type == LIBBPF_MAP_BSS) 1586 return 0; 1587 1588 data = map->libbpf_type == LIBBPF_MAP_DATA ? 1589 obj->sections.data : obj->sections.rodata; 1590 1591 err = bpf_map_update_elem(map->fd, &zero, data, 0); 1592 /* Freeze .rodata map as read-only from syscall side. */ 1593 if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { 1594 err = bpf_map_freeze(map->fd); 1595 if (err) { 1596 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1597 pr_warning("Error freezing map(%s) as read-only: %s\n", 1598 map->name, cp); 1599 err = 0; 1600 } 1601 } 1602 return err; 1603 } 1604 1605 static int 1606 bpf_object__create_maps(struct bpf_object *obj) 1607 { 1608 struct bpf_create_map_attr create_attr = {}; 1609 unsigned int i; 1610 int err; 1611 1612 for (i = 0; i < obj->nr_maps; i++) { 1613 struct bpf_map *map = &obj->maps[i]; 1614 struct bpf_map_def *def = &map->def; 1615 char *cp, errmsg[STRERR_BUFSIZE]; 1616 int *pfd = &map->fd; 1617 1618 if (map->fd >= 0) { 1619 pr_debug("skip map create (preset) %s: fd=%d\n", 1620 map->name, map->fd); 1621 continue; 1622 } 1623 1624 if (obj->caps.name) 1625 create_attr.name = map->name; 1626 create_attr.map_ifindex = map->map_ifindex; 1627 create_attr.map_type = def->type; 1628 create_attr.map_flags = def->map_flags; 1629 create_attr.key_size = def->key_size; 1630 create_attr.value_size = def->value_size; 1631 create_attr.max_entries = def->max_entries; 1632 create_attr.btf_fd = 0; 1633 create_attr.btf_key_type_id = 0; 1634 create_attr.btf_value_type_id = 0; 1635 if (bpf_map_type__is_map_in_map(def->type) && 1636 map->inner_map_fd >= 0) 1637 create_attr.inner_map_fd = map->inner_map_fd; 1638 1639 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) { 1640 create_attr.btf_fd = btf__fd(obj->btf); 1641 create_attr.btf_key_type_id = map->btf_key_type_id; 1642 create_attr.btf_value_type_id = map->btf_value_type_id; 1643 } 1644 1645 *pfd = bpf_create_map_xattr(&create_attr); 1646 if (*pfd < 0 && create_attr.btf_key_type_id) { 1647 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1648 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 1649 map->name, cp, errno); 1650 create_attr.btf_fd = 0; 1651 create_attr.btf_key_type_id = 0; 1652 create_attr.btf_value_type_id = 0; 1653 map->btf_key_type_id = 0; 1654 map->btf_value_type_id = 0; 1655 *pfd = bpf_create_map_xattr(&create_attr); 1656 } 1657 1658 if (*pfd < 0) { 1659 size_t j; 1660 1661 err = *pfd; 1662 err_out: 1663 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1664 pr_warning("failed to create map (name: '%s'): %s\n", 1665 map->name, cp); 1666 for (j = 0; j < i; j++) 1667 zclose(obj->maps[j].fd); 1668 return err; 1669 } 1670 1671 if (bpf_map__is_internal(map)) { 1672 err = bpf_object__populate_internal_map(obj, map); 1673 if (err < 0) { 1674 zclose(*pfd); 1675 goto err_out; 1676 } 1677 } 1678 1679 pr_debug("create map %s: fd=%d\n", map->name, *pfd); 1680 } 1681 1682 return 0; 1683 } 1684 1685 static int 1686 check_btf_ext_reloc_err(struct bpf_program *prog, int err, 1687 void *btf_prog_info, const char *info_name) 1688 { 1689 if (err != -ENOENT) { 1690 pr_warning("Error in loading %s for sec %s.\n", 1691 info_name, prog->section_name); 1692 return err; 1693 } 1694 1695 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ 1696 1697 if (btf_prog_info) { 1698 /* 1699 * Some info has already been found but has problem 1700 * in the last btf_ext reloc. Must have to error 1701 * out. 1702 */ 1703 pr_warning("Error in relocating %s for sec %s.\n", 1704 info_name, prog->section_name); 1705 return err; 1706 } 1707 1708 /* 1709 * Have problem loading the very first info. Ignore 1710 * the rest. 1711 */ 1712 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", 1713 info_name, prog->section_name, info_name); 1714 return 0; 1715 } 1716 1717 static int 1718 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, 1719 const char *section_name, __u32 insn_offset) 1720 { 1721 int err; 1722 1723 if (!insn_offset || prog->func_info) { 1724 /* 1725 * !insn_offset => main program 1726 * 1727 * For sub prog, the main program's func_info has to 1728 * be loaded first (i.e. prog->func_info != NULL) 1729 */ 1730 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, 1731 section_name, insn_offset, 1732 &prog->func_info, 1733 &prog->func_info_cnt); 1734 if (err) 1735 return check_btf_ext_reloc_err(prog, err, 1736 prog->func_info, 1737 "bpf_func_info"); 1738 1739 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); 1740 } 1741 1742 if (!insn_offset || prog->line_info) { 1743 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, 1744 section_name, insn_offset, 1745 &prog->line_info, 1746 &prog->line_info_cnt); 1747 if (err) 1748 return check_btf_ext_reloc_err(prog, err, 1749 prog->line_info, 1750 "bpf_line_info"); 1751 1752 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); 1753 } 1754 1755 if (!insn_offset) 1756 prog->btf_fd = btf__fd(obj->btf); 1757 1758 return 0; 1759 } 1760 1761 static int 1762 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 1763 struct reloc_desc *relo) 1764 { 1765 struct bpf_insn *insn, *new_insn; 1766 struct bpf_program *text; 1767 size_t new_cnt; 1768 int err; 1769 1770 if (relo->type != RELO_CALL) 1771 return -LIBBPF_ERRNO__RELOC; 1772 1773 if (prog->idx == obj->efile.text_shndx) { 1774 pr_warning("relo in .text insn %d into off %d\n", 1775 relo->insn_idx, relo->text_off); 1776 return -LIBBPF_ERRNO__RELOC; 1777 } 1778 1779 if (prog->main_prog_cnt == 0) { 1780 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 1781 if (!text) { 1782 pr_warning("no .text section found yet relo into text exist\n"); 1783 return -LIBBPF_ERRNO__RELOC; 1784 } 1785 new_cnt = prog->insns_cnt + text->insns_cnt; 1786 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); 1787 if (!new_insn) { 1788 pr_warning("oom in prog realloc\n"); 1789 return -ENOMEM; 1790 } 1791 1792 if (obj->btf_ext) { 1793 err = bpf_program_reloc_btf_ext(prog, obj, 1794 text->section_name, 1795 prog->insns_cnt); 1796 if (err) 1797 return err; 1798 } 1799 1800 memcpy(new_insn + prog->insns_cnt, text->insns, 1801 text->insns_cnt * sizeof(*insn)); 1802 prog->insns = new_insn; 1803 prog->main_prog_cnt = prog->insns_cnt; 1804 prog->insns_cnt = new_cnt; 1805 pr_debug("added %zd insn from %s to prog %s\n", 1806 text->insns_cnt, text->section_name, 1807 prog->section_name); 1808 } 1809 insn = &prog->insns[relo->insn_idx]; 1810 insn->imm += prog->main_prog_cnt - relo->insn_idx; 1811 return 0; 1812 } 1813 1814 static int 1815 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1816 { 1817 int i, err; 1818 1819 if (!prog) 1820 return 0; 1821 1822 if (obj->btf_ext) { 1823 err = bpf_program_reloc_btf_ext(prog, obj, 1824 prog->section_name, 0); 1825 if (err) 1826 return err; 1827 } 1828 1829 if (!prog->reloc_desc) 1830 return 0; 1831 1832 for (i = 0; i < prog->nr_reloc; i++) { 1833 if (prog->reloc_desc[i].type == RELO_LD64 || 1834 prog->reloc_desc[i].type == RELO_DATA) { 1835 bool relo_data = prog->reloc_desc[i].type == RELO_DATA; 1836 struct bpf_insn *insns = prog->insns; 1837 int insn_idx, map_idx; 1838 1839 insn_idx = prog->reloc_desc[i].insn_idx; 1840 map_idx = prog->reloc_desc[i].map_idx; 1841 1842 if (insn_idx + 1 >= (int)prog->insns_cnt) { 1843 pr_warning("relocation out of range: '%s'\n", 1844 prog->section_name); 1845 return -LIBBPF_ERRNO__RELOC; 1846 } 1847 1848 if (!relo_data) { 1849 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1850 } else { 1851 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; 1852 insns[insn_idx + 1].imm = insns[insn_idx].imm; 1853 } 1854 insns[insn_idx].imm = obj->maps[map_idx].fd; 1855 } else if (prog->reloc_desc[i].type == RELO_CALL) { 1856 err = bpf_program__reloc_text(prog, obj, 1857 &prog->reloc_desc[i]); 1858 if (err) 1859 return err; 1860 } 1861 } 1862 1863 zfree(&prog->reloc_desc); 1864 prog->nr_reloc = 0; 1865 return 0; 1866 } 1867 1868 1869 static int 1870 bpf_object__relocate(struct bpf_object *obj) 1871 { 1872 struct bpf_program *prog; 1873 size_t i; 1874 int err; 1875 1876 for (i = 0; i < obj->nr_programs; i++) { 1877 prog = &obj->programs[i]; 1878 1879 err = bpf_program__relocate(prog, obj); 1880 if (err) { 1881 pr_warning("failed to relocate '%s'\n", 1882 prog->section_name); 1883 return err; 1884 } 1885 } 1886 return 0; 1887 } 1888 1889 static int bpf_object__collect_reloc(struct bpf_object *obj) 1890 { 1891 int i, err; 1892 1893 if (!obj_elf_valid(obj)) { 1894 pr_warning("Internal error: elf object is closed\n"); 1895 return -LIBBPF_ERRNO__INTERNAL; 1896 } 1897 1898 for (i = 0; i < obj->efile.nr_reloc; i++) { 1899 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 1900 Elf_Data *data = obj->efile.reloc[i].data; 1901 int idx = shdr->sh_info; 1902 struct bpf_program *prog; 1903 1904 if (shdr->sh_type != SHT_REL) { 1905 pr_warning("internal error at %d\n", __LINE__); 1906 return -LIBBPF_ERRNO__INTERNAL; 1907 } 1908 1909 prog = bpf_object__find_prog_by_idx(obj, idx); 1910 if (!prog) { 1911 pr_warning("relocation failed: no section(%d)\n", idx); 1912 return -LIBBPF_ERRNO__RELOC; 1913 } 1914 1915 err = bpf_program__collect_reloc(prog, 1916 shdr, data, 1917 obj); 1918 if (err) 1919 return err; 1920 } 1921 return 0; 1922 } 1923 1924 static int 1925 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, 1926 char *license, __u32 kern_version, int *pfd) 1927 { 1928 struct bpf_load_program_attr load_attr; 1929 char *cp, errmsg[STRERR_BUFSIZE]; 1930 int log_buf_size = BPF_LOG_BUF_SIZE; 1931 char *log_buf; 1932 int ret; 1933 1934 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); 1935 load_attr.prog_type = prog->type; 1936 load_attr.expected_attach_type = prog->expected_attach_type; 1937 if (prog->caps->name) 1938 load_attr.name = prog->name; 1939 load_attr.insns = insns; 1940 load_attr.insns_cnt = insns_cnt; 1941 load_attr.license = license; 1942 load_attr.kern_version = kern_version; 1943 load_attr.prog_ifindex = prog->prog_ifindex; 1944 load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; 1945 load_attr.func_info = prog->func_info; 1946 load_attr.func_info_rec_size = prog->func_info_rec_size; 1947 load_attr.func_info_cnt = prog->func_info_cnt; 1948 load_attr.line_info = prog->line_info; 1949 load_attr.line_info_rec_size = prog->line_info_rec_size; 1950 load_attr.line_info_cnt = prog->line_info_cnt; 1951 load_attr.log_level = prog->log_level; 1952 if (!load_attr.insns || !load_attr.insns_cnt) 1953 return -EINVAL; 1954 1955 retry_load: 1956 log_buf = malloc(log_buf_size); 1957 if (!log_buf) 1958 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 1959 1960 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); 1961 1962 if (ret >= 0) { 1963 if (load_attr.log_level) 1964 pr_debug("verifier log:\n%s", log_buf); 1965 *pfd = ret; 1966 ret = 0; 1967 goto out; 1968 } 1969 1970 if (errno == ENOSPC) { 1971 log_buf_size <<= 1; 1972 free(log_buf); 1973 goto retry_load; 1974 } 1975 ret = -LIBBPF_ERRNO__LOAD; 1976 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1977 pr_warning("load bpf program failed: %s\n", cp); 1978 1979 if (log_buf && log_buf[0] != '\0') { 1980 ret = -LIBBPF_ERRNO__VERIFY; 1981 pr_warning("-- BEGIN DUMP LOG ---\n"); 1982 pr_warning("\n%s\n", log_buf); 1983 pr_warning("-- END LOG --\n"); 1984 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { 1985 pr_warning("Program too large (%zu insns), at most %d insns\n", 1986 load_attr.insns_cnt, BPF_MAXINSNS); 1987 ret = -LIBBPF_ERRNO__PROG2BIG; 1988 } else { 1989 /* Wrong program type? */ 1990 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { 1991 int fd; 1992 1993 load_attr.prog_type = BPF_PROG_TYPE_KPROBE; 1994 load_attr.expected_attach_type = 0; 1995 fd = bpf_load_program_xattr(&load_attr, NULL, 0); 1996 if (fd >= 0) { 1997 close(fd); 1998 ret = -LIBBPF_ERRNO__PROGTYPE; 1999 goto out; 2000 } 2001 } 2002 2003 if (log_buf) 2004 ret = -LIBBPF_ERRNO__KVER; 2005 } 2006 2007 out: 2008 free(log_buf); 2009 return ret; 2010 } 2011 2012 int 2013 bpf_program__load(struct bpf_program *prog, 2014 char *license, __u32 kern_version) 2015 { 2016 int err = 0, fd, i; 2017 2018 if (prog->instances.nr < 0 || !prog->instances.fds) { 2019 if (prog->preprocessor) { 2020 pr_warning("Internal error: can't load program '%s'\n", 2021 prog->section_name); 2022 return -LIBBPF_ERRNO__INTERNAL; 2023 } 2024 2025 prog->instances.fds = malloc(sizeof(int)); 2026 if (!prog->instances.fds) { 2027 pr_warning("Not enough memory for BPF fds\n"); 2028 return -ENOMEM; 2029 } 2030 prog->instances.nr = 1; 2031 prog->instances.fds[0] = -1; 2032 } 2033 2034 if (!prog->preprocessor) { 2035 if (prog->instances.nr != 1) { 2036 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 2037 prog->section_name, prog->instances.nr); 2038 } 2039 err = load_program(prog, prog->insns, prog->insns_cnt, 2040 license, kern_version, &fd); 2041 if (!err) 2042 prog->instances.fds[0] = fd; 2043 goto out; 2044 } 2045 2046 for (i = 0; i < prog->instances.nr; i++) { 2047 struct bpf_prog_prep_result result; 2048 bpf_program_prep_t preprocessor = prog->preprocessor; 2049 2050 memset(&result, 0, sizeof(result)); 2051 err = preprocessor(prog, i, prog->insns, 2052 prog->insns_cnt, &result); 2053 if (err) { 2054 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 2055 i, prog->section_name); 2056 goto out; 2057 } 2058 2059 if (!result.new_insn_ptr || !result.new_insn_cnt) { 2060 pr_debug("Skip loading the %dth instance of program '%s'\n", 2061 i, prog->section_name); 2062 prog->instances.fds[i] = -1; 2063 if (result.pfd) 2064 *result.pfd = -1; 2065 continue; 2066 } 2067 2068 err = load_program(prog, result.new_insn_ptr, 2069 result.new_insn_cnt, 2070 license, kern_version, &fd); 2071 2072 if (err) { 2073 pr_warning("Loading the %dth instance of program '%s' failed\n", 2074 i, prog->section_name); 2075 goto out; 2076 } 2077 2078 if (result.pfd) 2079 *result.pfd = fd; 2080 prog->instances.fds[i] = fd; 2081 } 2082 out: 2083 if (err) 2084 pr_warning("failed to load program '%s'\n", 2085 prog->section_name); 2086 zfree(&prog->insns); 2087 prog->insns_cnt = 0; 2088 return err; 2089 } 2090 2091 static bool bpf_program__is_function_storage(struct bpf_program *prog, 2092 struct bpf_object *obj) 2093 { 2094 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; 2095 } 2096 2097 static int 2098 bpf_object__load_progs(struct bpf_object *obj) 2099 { 2100 size_t i; 2101 int err; 2102 2103 for (i = 0; i < obj->nr_programs; i++) { 2104 if (bpf_program__is_function_storage(&obj->programs[i], obj)) 2105 continue; 2106 err = bpf_program__load(&obj->programs[i], 2107 obj->license, 2108 obj->kern_version); 2109 if (err) 2110 return err; 2111 } 2112 return 0; 2113 } 2114 2115 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) 2116 { 2117 switch (type) { 2118 case BPF_PROG_TYPE_SOCKET_FILTER: 2119 case BPF_PROG_TYPE_SCHED_CLS: 2120 case BPF_PROG_TYPE_SCHED_ACT: 2121 case BPF_PROG_TYPE_XDP: 2122 case BPF_PROG_TYPE_CGROUP_SKB: 2123 case BPF_PROG_TYPE_CGROUP_SOCK: 2124 case BPF_PROG_TYPE_LWT_IN: 2125 case BPF_PROG_TYPE_LWT_OUT: 2126 case BPF_PROG_TYPE_LWT_XMIT: 2127 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 2128 case BPF_PROG_TYPE_SOCK_OPS: 2129 case BPF_PROG_TYPE_SK_SKB: 2130 case BPF_PROG_TYPE_CGROUP_DEVICE: 2131 case BPF_PROG_TYPE_SK_MSG: 2132 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 2133 case BPF_PROG_TYPE_LIRC_MODE2: 2134 case BPF_PROG_TYPE_SK_REUSEPORT: 2135 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2136 case BPF_PROG_TYPE_UNSPEC: 2137 case BPF_PROG_TYPE_TRACEPOINT: 2138 case BPF_PROG_TYPE_RAW_TRACEPOINT: 2139 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 2140 case BPF_PROG_TYPE_PERF_EVENT: 2141 case BPF_PROG_TYPE_CGROUP_SYSCTL: 2142 return false; 2143 case BPF_PROG_TYPE_KPROBE: 2144 default: 2145 return true; 2146 } 2147 } 2148 2149 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) 2150 { 2151 if (needs_kver && obj->kern_version == 0) { 2152 pr_warning("%s doesn't provide kernel version\n", 2153 obj->path); 2154 return -LIBBPF_ERRNO__KVERSION; 2155 } 2156 return 0; 2157 } 2158 2159 static struct bpf_object * 2160 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, 2161 bool needs_kver, int flags) 2162 { 2163 struct bpf_object *obj; 2164 int err; 2165 2166 if (elf_version(EV_CURRENT) == EV_NONE) { 2167 pr_warning("failed to init libelf for %s\n", path); 2168 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 2169 } 2170 2171 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 2172 if (IS_ERR(obj)) 2173 return obj; 2174 2175 CHECK_ERR(bpf_object__elf_init(obj), err, out); 2176 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 2177 CHECK_ERR(bpf_object__probe_caps(obj), err, out); 2178 CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); 2179 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 2180 CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); 2181 2182 bpf_object__elf_finish(obj); 2183 return obj; 2184 out: 2185 bpf_object__close(obj); 2186 return ERR_PTR(err); 2187 } 2188 2189 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, 2190 int flags) 2191 { 2192 /* param validation */ 2193 if (!attr->file) 2194 return NULL; 2195 2196 pr_debug("loading %s\n", attr->file); 2197 2198 return __bpf_object__open(attr->file, NULL, 0, 2199 bpf_prog_type__needs_kver(attr->prog_type), 2200 flags); 2201 } 2202 2203 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) 2204 { 2205 return __bpf_object__open_xattr(attr, 0); 2206 } 2207 2208 struct bpf_object *bpf_object__open(const char *path) 2209 { 2210 struct bpf_object_open_attr attr = { 2211 .file = path, 2212 .prog_type = BPF_PROG_TYPE_UNSPEC, 2213 }; 2214 2215 return bpf_object__open_xattr(&attr); 2216 } 2217 2218 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 2219 size_t obj_buf_sz, 2220 const char *name) 2221 { 2222 char tmp_name[64]; 2223 2224 /* param validation */ 2225 if (!obj_buf || obj_buf_sz <= 0) 2226 return NULL; 2227 2228 if (!name) { 2229 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 2230 (unsigned long)obj_buf, 2231 (unsigned long)obj_buf_sz); 2232 tmp_name[sizeof(tmp_name) - 1] = '\0'; 2233 name = tmp_name; 2234 } 2235 pr_debug("loading object '%s' from buffer\n", 2236 name); 2237 2238 return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); 2239 } 2240 2241 int bpf_object__unload(struct bpf_object *obj) 2242 { 2243 size_t i; 2244 2245 if (!obj) 2246 return -EINVAL; 2247 2248 for (i = 0; i < obj->nr_maps; i++) 2249 zclose(obj->maps[i].fd); 2250 2251 for (i = 0; i < obj->nr_programs; i++) 2252 bpf_program__unload(&obj->programs[i]); 2253 2254 return 0; 2255 } 2256 2257 int bpf_object__load(struct bpf_object *obj) 2258 { 2259 int err; 2260 2261 if (!obj) 2262 return -EINVAL; 2263 2264 if (obj->loaded) { 2265 pr_warning("object should not be loaded twice\n"); 2266 return -EINVAL; 2267 } 2268 2269 obj->loaded = true; 2270 2271 CHECK_ERR(bpf_object__create_maps(obj), err, out); 2272 CHECK_ERR(bpf_object__relocate(obj), err, out); 2273 CHECK_ERR(bpf_object__load_progs(obj), err, out); 2274 2275 return 0; 2276 out: 2277 bpf_object__unload(obj); 2278 pr_warning("failed to load object '%s'\n", obj->path); 2279 return err; 2280 } 2281 2282 static int check_path(const char *path) 2283 { 2284 char *cp, errmsg[STRERR_BUFSIZE]; 2285 struct statfs st_fs; 2286 char *dname, *dir; 2287 int err = 0; 2288 2289 if (path == NULL) 2290 return -EINVAL; 2291 2292 dname = strdup(path); 2293 if (dname == NULL) 2294 return -ENOMEM; 2295 2296 dir = dirname(dname); 2297 if (statfs(dir, &st_fs)) { 2298 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2299 pr_warning("failed to statfs %s: %s\n", dir, cp); 2300 err = -errno; 2301 } 2302 free(dname); 2303 2304 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 2305 pr_warning("specified path %s is not on BPF FS\n", path); 2306 err = -EINVAL; 2307 } 2308 2309 return err; 2310 } 2311 2312 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 2313 int instance) 2314 { 2315 char *cp, errmsg[STRERR_BUFSIZE]; 2316 int err; 2317 2318 err = check_path(path); 2319 if (err) 2320 return err; 2321 2322 if (prog == NULL) { 2323 pr_warning("invalid program pointer\n"); 2324 return -EINVAL; 2325 } 2326 2327 if (instance < 0 || instance >= prog->instances.nr) { 2328 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2329 instance, prog->section_name, prog->instances.nr); 2330 return -EINVAL; 2331 } 2332 2333 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 2334 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2335 pr_warning("failed to pin program: %s\n", cp); 2336 return -errno; 2337 } 2338 pr_debug("pinned program '%s'\n", path); 2339 2340 return 0; 2341 } 2342 2343 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, 2344 int instance) 2345 { 2346 int err; 2347 2348 err = check_path(path); 2349 if (err) 2350 return err; 2351 2352 if (prog == NULL) { 2353 pr_warning("invalid program pointer\n"); 2354 return -EINVAL; 2355 } 2356 2357 if (instance < 0 || instance >= prog->instances.nr) { 2358 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 2359 instance, prog->section_name, prog->instances.nr); 2360 return -EINVAL; 2361 } 2362 2363 err = unlink(path); 2364 if (err != 0) 2365 return -errno; 2366 pr_debug("unpinned program '%s'\n", path); 2367 2368 return 0; 2369 } 2370 2371 static int make_dir(const char *path) 2372 { 2373 char *cp, errmsg[STRERR_BUFSIZE]; 2374 int err = 0; 2375 2376 if (mkdir(path, 0700) && errno != EEXIST) 2377 err = -errno; 2378 2379 if (err) { 2380 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 2381 pr_warning("failed to mkdir %s: %s\n", path, cp); 2382 } 2383 return err; 2384 } 2385 2386 int bpf_program__pin(struct bpf_program *prog, const char *path) 2387 { 2388 int i, err; 2389 2390 err = check_path(path); 2391 if (err) 2392 return err; 2393 2394 if (prog == NULL) { 2395 pr_warning("invalid program pointer\n"); 2396 return -EINVAL; 2397 } 2398 2399 if (prog->instances.nr <= 0) { 2400 pr_warning("no instances of prog %s to pin\n", 2401 prog->section_name); 2402 return -EINVAL; 2403 } 2404 2405 if (prog->instances.nr == 1) { 2406 /* don't create subdirs when pinning single instance */ 2407 return bpf_program__pin_instance(prog, path, 0); 2408 } 2409 2410 err = make_dir(path); 2411 if (err) 2412 return err; 2413 2414 for (i = 0; i < prog->instances.nr; i++) { 2415 char buf[PATH_MAX]; 2416 int len; 2417 2418 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2419 if (len < 0) { 2420 err = -EINVAL; 2421 goto err_unpin; 2422 } else if (len >= PATH_MAX) { 2423 err = -ENAMETOOLONG; 2424 goto err_unpin; 2425 } 2426 2427 err = bpf_program__pin_instance(prog, buf, i); 2428 if (err) 2429 goto err_unpin; 2430 } 2431 2432 return 0; 2433 2434 err_unpin: 2435 for (i = i - 1; i >= 0; i--) { 2436 char buf[PATH_MAX]; 2437 int len; 2438 2439 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2440 if (len < 0) 2441 continue; 2442 else if (len >= PATH_MAX) 2443 continue; 2444 2445 bpf_program__unpin_instance(prog, buf, i); 2446 } 2447 2448 rmdir(path); 2449 2450 return err; 2451 } 2452 2453 int bpf_program__unpin(struct bpf_program *prog, const char *path) 2454 { 2455 int i, err; 2456 2457 err = check_path(path); 2458 if (err) 2459 return err; 2460 2461 if (prog == NULL) { 2462 pr_warning("invalid program pointer\n"); 2463 return -EINVAL; 2464 } 2465 2466 if (prog->instances.nr <= 0) { 2467 pr_warning("no instances of prog %s to pin\n", 2468 prog->section_name); 2469 return -EINVAL; 2470 } 2471 2472 if (prog->instances.nr == 1) { 2473 /* don't create subdirs when pinning single instance */ 2474 return bpf_program__unpin_instance(prog, path, 0); 2475 } 2476 2477 for (i = 0; i < prog->instances.nr; i++) { 2478 char buf[PATH_MAX]; 2479 int len; 2480 2481 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 2482 if (len < 0) 2483 return -EINVAL; 2484 else if (len >= PATH_MAX) 2485 return -ENAMETOOLONG; 2486 2487 err = bpf_program__unpin_instance(prog, buf, i); 2488 if (err) 2489 return err; 2490 } 2491 2492 err = rmdir(path); 2493 if (err) 2494 return -errno; 2495 2496 return 0; 2497 } 2498 2499 int bpf_map__pin(struct bpf_map *map, const char *path) 2500 { 2501 char *cp, errmsg[STRERR_BUFSIZE]; 2502 int err; 2503 2504 err = check_path(path); 2505 if (err) 2506 return err; 2507 2508 if (map == NULL) { 2509 pr_warning("invalid map pointer\n"); 2510 return -EINVAL; 2511 } 2512 2513 if (bpf_obj_pin(map->fd, path)) { 2514 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 2515 pr_warning("failed to pin map: %s\n", cp); 2516 return -errno; 2517 } 2518 2519 pr_debug("pinned map '%s'\n", path); 2520 2521 return 0; 2522 } 2523 2524 int bpf_map__unpin(struct bpf_map *map, const char *path) 2525 { 2526 int err; 2527 2528 err = check_path(path); 2529 if (err) 2530 return err; 2531 2532 if (map == NULL) { 2533 pr_warning("invalid map pointer\n"); 2534 return -EINVAL; 2535 } 2536 2537 err = unlink(path); 2538 if (err != 0) 2539 return -errno; 2540 pr_debug("unpinned map '%s'\n", path); 2541 2542 return 0; 2543 } 2544 2545 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 2546 { 2547 struct bpf_map *map; 2548 int err; 2549 2550 if (!obj) 2551 return -ENOENT; 2552 2553 if (!obj->loaded) { 2554 pr_warning("object not yet loaded; load it first\n"); 2555 return -ENOENT; 2556 } 2557 2558 err = make_dir(path); 2559 if (err) 2560 return err; 2561 2562 bpf_object__for_each_map(map, obj) { 2563 char buf[PATH_MAX]; 2564 int len; 2565 2566 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2567 bpf_map__name(map)); 2568 if (len < 0) { 2569 err = -EINVAL; 2570 goto err_unpin_maps; 2571 } else if (len >= PATH_MAX) { 2572 err = -ENAMETOOLONG; 2573 goto err_unpin_maps; 2574 } 2575 2576 err = bpf_map__pin(map, buf); 2577 if (err) 2578 goto err_unpin_maps; 2579 } 2580 2581 return 0; 2582 2583 err_unpin_maps: 2584 while ((map = bpf_map__prev(map, obj))) { 2585 char buf[PATH_MAX]; 2586 int len; 2587 2588 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2589 bpf_map__name(map)); 2590 if (len < 0) 2591 continue; 2592 else if (len >= PATH_MAX) 2593 continue; 2594 2595 bpf_map__unpin(map, buf); 2596 } 2597 2598 return err; 2599 } 2600 2601 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 2602 { 2603 struct bpf_map *map; 2604 int err; 2605 2606 if (!obj) 2607 return -ENOENT; 2608 2609 bpf_object__for_each_map(map, obj) { 2610 char buf[PATH_MAX]; 2611 int len; 2612 2613 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2614 bpf_map__name(map)); 2615 if (len < 0) 2616 return -EINVAL; 2617 else if (len >= PATH_MAX) 2618 return -ENAMETOOLONG; 2619 2620 err = bpf_map__unpin(map, buf); 2621 if (err) 2622 return err; 2623 } 2624 2625 return 0; 2626 } 2627 2628 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 2629 { 2630 struct bpf_program *prog; 2631 int err; 2632 2633 if (!obj) 2634 return -ENOENT; 2635 2636 if (!obj->loaded) { 2637 pr_warning("object not yet loaded; load it first\n"); 2638 return -ENOENT; 2639 } 2640 2641 err = make_dir(path); 2642 if (err) 2643 return err; 2644 2645 bpf_object__for_each_program(prog, obj) { 2646 char buf[PATH_MAX]; 2647 int len; 2648 2649 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2650 prog->pin_name); 2651 if (len < 0) { 2652 err = -EINVAL; 2653 goto err_unpin_programs; 2654 } else if (len >= PATH_MAX) { 2655 err = -ENAMETOOLONG; 2656 goto err_unpin_programs; 2657 } 2658 2659 err = bpf_program__pin(prog, buf); 2660 if (err) 2661 goto err_unpin_programs; 2662 } 2663 2664 return 0; 2665 2666 err_unpin_programs: 2667 while ((prog = bpf_program__prev(prog, obj))) { 2668 char buf[PATH_MAX]; 2669 int len; 2670 2671 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2672 prog->pin_name); 2673 if (len < 0) 2674 continue; 2675 else if (len >= PATH_MAX) 2676 continue; 2677 2678 bpf_program__unpin(prog, buf); 2679 } 2680 2681 return err; 2682 } 2683 2684 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 2685 { 2686 struct bpf_program *prog; 2687 int err; 2688 2689 if (!obj) 2690 return -ENOENT; 2691 2692 bpf_object__for_each_program(prog, obj) { 2693 char buf[PATH_MAX]; 2694 int len; 2695 2696 len = snprintf(buf, PATH_MAX, "%s/%s", path, 2697 prog->pin_name); 2698 if (len < 0) 2699 return -EINVAL; 2700 else if (len >= PATH_MAX) 2701 return -ENAMETOOLONG; 2702 2703 err = bpf_program__unpin(prog, buf); 2704 if (err) 2705 return err; 2706 } 2707 2708 return 0; 2709 } 2710 2711 int bpf_object__pin(struct bpf_object *obj, const char *path) 2712 { 2713 int err; 2714 2715 err = bpf_object__pin_maps(obj, path); 2716 if (err) 2717 return err; 2718 2719 err = bpf_object__pin_programs(obj, path); 2720 if (err) { 2721 bpf_object__unpin_maps(obj, path); 2722 return err; 2723 } 2724 2725 return 0; 2726 } 2727 2728 void bpf_object__close(struct bpf_object *obj) 2729 { 2730 size_t i; 2731 2732 if (!obj) 2733 return; 2734 2735 if (obj->clear_priv) 2736 obj->clear_priv(obj, obj->priv); 2737 2738 bpf_object__elf_finish(obj); 2739 bpf_object__unload(obj); 2740 btf__free(obj->btf); 2741 btf_ext__free(obj->btf_ext); 2742 2743 for (i = 0; i < obj->nr_maps; i++) { 2744 zfree(&obj->maps[i].name); 2745 if (obj->maps[i].clear_priv) 2746 obj->maps[i].clear_priv(&obj->maps[i], 2747 obj->maps[i].priv); 2748 obj->maps[i].priv = NULL; 2749 obj->maps[i].clear_priv = NULL; 2750 } 2751 2752 zfree(&obj->sections.rodata); 2753 zfree(&obj->sections.data); 2754 zfree(&obj->maps); 2755 obj->nr_maps = 0; 2756 2757 if (obj->programs && obj->nr_programs) { 2758 for (i = 0; i < obj->nr_programs; i++) 2759 bpf_program__exit(&obj->programs[i]); 2760 } 2761 zfree(&obj->programs); 2762 2763 list_del(&obj->list); 2764 free(obj); 2765 } 2766 2767 struct bpf_object * 2768 bpf_object__next(struct bpf_object *prev) 2769 { 2770 struct bpf_object *next; 2771 2772 if (!prev) 2773 next = list_first_entry(&bpf_objects_list, 2774 struct bpf_object, 2775 list); 2776 else 2777 next = list_next_entry(prev, list); 2778 2779 /* Empty list is noticed here so don't need checking on entry. */ 2780 if (&next->list == &bpf_objects_list) 2781 return NULL; 2782 2783 return next; 2784 } 2785 2786 const char *bpf_object__name(struct bpf_object *obj) 2787 { 2788 return obj ? obj->path : ERR_PTR(-EINVAL); 2789 } 2790 2791 unsigned int bpf_object__kversion(struct bpf_object *obj) 2792 { 2793 return obj ? obj->kern_version : 0; 2794 } 2795 2796 struct btf *bpf_object__btf(struct bpf_object *obj) 2797 { 2798 return obj ? obj->btf : NULL; 2799 } 2800 2801 int bpf_object__btf_fd(const struct bpf_object *obj) 2802 { 2803 return obj->btf ? btf__fd(obj->btf) : -1; 2804 } 2805 2806 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 2807 bpf_object_clear_priv_t clear_priv) 2808 { 2809 if (obj->priv && obj->clear_priv) 2810 obj->clear_priv(obj, obj->priv); 2811 2812 obj->priv = priv; 2813 obj->clear_priv = clear_priv; 2814 return 0; 2815 } 2816 2817 void *bpf_object__priv(struct bpf_object *obj) 2818 { 2819 return obj ? obj->priv : ERR_PTR(-EINVAL); 2820 } 2821 2822 static struct bpf_program * 2823 __bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward) 2824 { 2825 size_t nr_programs = obj->nr_programs; 2826 ssize_t idx; 2827 2828 if (!nr_programs) 2829 return NULL; 2830 2831 if (!p) 2832 /* Iter from the beginning */ 2833 return forward ? &obj->programs[0] : 2834 &obj->programs[nr_programs - 1]; 2835 2836 if (p->obj != obj) { 2837 pr_warning("error: program handler doesn't match object\n"); 2838 return NULL; 2839 } 2840 2841 idx = (p - obj->programs) + (forward ? 1 : -1); 2842 if (idx >= obj->nr_programs || idx < 0) 2843 return NULL; 2844 return &obj->programs[idx]; 2845 } 2846 2847 struct bpf_program * 2848 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 2849 { 2850 struct bpf_program *prog = prev; 2851 2852 do { 2853 prog = __bpf_program__iter(prog, obj, true); 2854 } while (prog && bpf_program__is_function_storage(prog, obj)); 2855 2856 return prog; 2857 } 2858 2859 struct bpf_program * 2860 bpf_program__prev(struct bpf_program *next, struct bpf_object *obj) 2861 { 2862 struct bpf_program *prog = next; 2863 2864 do { 2865 prog = __bpf_program__iter(prog, obj, false); 2866 } while (prog && bpf_program__is_function_storage(prog, obj)); 2867 2868 return prog; 2869 } 2870 2871 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 2872 bpf_program_clear_priv_t clear_priv) 2873 { 2874 if (prog->priv && prog->clear_priv) 2875 prog->clear_priv(prog, prog->priv); 2876 2877 prog->priv = priv; 2878 prog->clear_priv = clear_priv; 2879 return 0; 2880 } 2881 2882 void *bpf_program__priv(struct bpf_program *prog) 2883 { 2884 return prog ? prog->priv : ERR_PTR(-EINVAL); 2885 } 2886 2887 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 2888 { 2889 prog->prog_ifindex = ifindex; 2890 } 2891 2892 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 2893 { 2894 const char *title; 2895 2896 title = prog->section_name; 2897 if (needs_copy) { 2898 title = strdup(title); 2899 if (!title) { 2900 pr_warning("failed to strdup program title\n"); 2901 return ERR_PTR(-ENOMEM); 2902 } 2903 } 2904 2905 return title; 2906 } 2907 2908 int bpf_program__fd(struct bpf_program *prog) 2909 { 2910 return bpf_program__nth_fd(prog, 0); 2911 } 2912 2913 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 2914 bpf_program_prep_t prep) 2915 { 2916 int *instances_fds; 2917 2918 if (nr_instances <= 0 || !prep) 2919 return -EINVAL; 2920 2921 if (prog->instances.nr > 0 || prog->instances.fds) { 2922 pr_warning("Can't set pre-processor after loading\n"); 2923 return -EINVAL; 2924 } 2925 2926 instances_fds = malloc(sizeof(int) * nr_instances); 2927 if (!instances_fds) { 2928 pr_warning("alloc memory failed for fds\n"); 2929 return -ENOMEM; 2930 } 2931 2932 /* fill all fd with -1 */ 2933 memset(instances_fds, -1, sizeof(int) * nr_instances); 2934 2935 prog->instances.nr = nr_instances; 2936 prog->instances.fds = instances_fds; 2937 prog->preprocessor = prep; 2938 return 0; 2939 } 2940 2941 int bpf_program__nth_fd(struct bpf_program *prog, int n) 2942 { 2943 int fd; 2944 2945 if (!prog) 2946 return -EINVAL; 2947 2948 if (n >= prog->instances.nr || n < 0) { 2949 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 2950 n, prog->section_name, prog->instances.nr); 2951 return -EINVAL; 2952 } 2953 2954 fd = prog->instances.fds[n]; 2955 if (fd < 0) { 2956 pr_warning("%dth instance of program '%s' is invalid\n", 2957 n, prog->section_name); 2958 return -ENOENT; 2959 } 2960 2961 return fd; 2962 } 2963 2964 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 2965 { 2966 prog->type = type; 2967 } 2968 2969 static bool bpf_program__is_type(struct bpf_program *prog, 2970 enum bpf_prog_type type) 2971 { 2972 return prog ? (prog->type == type) : false; 2973 } 2974 2975 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 2976 int bpf_program__set_##NAME(struct bpf_program *prog) \ 2977 { \ 2978 if (!prog) \ 2979 return -EINVAL; \ 2980 bpf_program__set_type(prog, TYPE); \ 2981 return 0; \ 2982 } \ 2983 \ 2984 bool bpf_program__is_##NAME(struct bpf_program *prog) \ 2985 { \ 2986 return bpf_program__is_type(prog, TYPE); \ 2987 } \ 2988 2989 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 2990 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 2991 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 2992 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 2993 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 2994 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); 2995 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 2996 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 2997 2998 void bpf_program__set_expected_attach_type(struct bpf_program *prog, 2999 enum bpf_attach_type type) 3000 { 3001 prog->expected_attach_type = type; 3002 } 3003 3004 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ 3005 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } 3006 3007 /* Programs that can NOT be attached. */ 3008 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) 3009 3010 /* Programs that can be attached. */ 3011 #define BPF_APROG_SEC(string, ptype, atype) \ 3012 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) 3013 3014 /* Programs that must specify expected attach type at load time. */ 3015 #define BPF_EAPROG_SEC(string, ptype, eatype) \ 3016 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) 3017 3018 /* Programs that can be attached but attach type can't be identified by section 3019 * name. Kept for backward compatibility. 3020 */ 3021 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) 3022 3023 static const struct { 3024 const char *sec; 3025 size_t len; 3026 enum bpf_prog_type prog_type; 3027 enum bpf_attach_type expected_attach_type; 3028 int is_attachable; 3029 enum bpf_attach_type attach_type; 3030 } section_names[] = { 3031 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 3032 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 3033 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 3034 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 3035 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 3036 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 3037 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), 3038 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 3039 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 3040 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 3041 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 3042 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 3043 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), 3044 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, 3045 BPF_CGROUP_INET_INGRESS), 3046 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, 3047 BPF_CGROUP_INET_EGRESS), 3048 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 3049 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, 3050 BPF_CGROUP_INET_SOCK_CREATE), 3051 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, 3052 BPF_CGROUP_INET4_POST_BIND), 3053 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, 3054 BPF_CGROUP_INET6_POST_BIND), 3055 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, 3056 BPF_CGROUP_DEVICE), 3057 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, 3058 BPF_CGROUP_SOCK_OPS), 3059 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, 3060 BPF_SK_SKB_STREAM_PARSER), 3061 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, 3062 BPF_SK_SKB_STREAM_VERDICT), 3063 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), 3064 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, 3065 BPF_SK_MSG_VERDICT), 3066 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, 3067 BPF_LIRC_MODE2), 3068 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, 3069 BPF_FLOW_DISSECTOR), 3070 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3071 BPF_CGROUP_INET4_BIND), 3072 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3073 BPF_CGROUP_INET6_BIND), 3074 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3075 BPF_CGROUP_INET4_CONNECT), 3076 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3077 BPF_CGROUP_INET6_CONNECT), 3078 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3079 BPF_CGROUP_UDP4_SENDMSG), 3080 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, 3081 BPF_CGROUP_UDP6_SENDMSG), 3082 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, 3083 BPF_CGROUP_SYSCTL), 3084 }; 3085 3086 #undef BPF_PROG_SEC_IMPL 3087 #undef BPF_PROG_SEC 3088 #undef BPF_APROG_SEC 3089 #undef BPF_EAPROG_SEC 3090 #undef BPF_APROG_COMPAT 3091 3092 #define MAX_TYPE_NAME_SIZE 32 3093 3094 static char *libbpf_get_type_names(bool attach_type) 3095 { 3096 int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; 3097 char *buf; 3098 3099 buf = malloc(len); 3100 if (!buf) 3101 return NULL; 3102 3103 buf[0] = '\0'; 3104 /* Forge string buf with all available names */ 3105 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3106 if (attach_type && !section_names[i].is_attachable) 3107 continue; 3108 3109 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { 3110 free(buf); 3111 return NULL; 3112 } 3113 strcat(buf, " "); 3114 strcat(buf, section_names[i].sec); 3115 } 3116 3117 return buf; 3118 } 3119 3120 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 3121 enum bpf_attach_type *expected_attach_type) 3122 { 3123 char *type_names; 3124 int i; 3125 3126 if (!name) 3127 return -EINVAL; 3128 3129 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3130 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3131 continue; 3132 *prog_type = section_names[i].prog_type; 3133 *expected_attach_type = section_names[i].expected_attach_type; 3134 return 0; 3135 } 3136 pr_warning("failed to guess program type based on ELF section name '%s'\n", name); 3137 type_names = libbpf_get_type_names(false); 3138 if (type_names != NULL) { 3139 pr_info("supported section(type) names are:%s\n", type_names); 3140 free(type_names); 3141 } 3142 3143 return -EINVAL; 3144 } 3145 3146 int libbpf_attach_type_by_name(const char *name, 3147 enum bpf_attach_type *attach_type) 3148 { 3149 char *type_names; 3150 int i; 3151 3152 if (!name) 3153 return -EINVAL; 3154 3155 for (i = 0; i < ARRAY_SIZE(section_names); i++) { 3156 if (strncmp(name, section_names[i].sec, section_names[i].len)) 3157 continue; 3158 if (!section_names[i].is_attachable) 3159 return -EINVAL; 3160 *attach_type = section_names[i].attach_type; 3161 return 0; 3162 } 3163 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); 3164 type_names = libbpf_get_type_names(true); 3165 if (type_names != NULL) { 3166 pr_info("attachable section(type) names are:%s\n", type_names); 3167 free(type_names); 3168 } 3169 3170 return -EINVAL; 3171 } 3172 3173 static int 3174 bpf_program__identify_section(struct bpf_program *prog, 3175 enum bpf_prog_type *prog_type, 3176 enum bpf_attach_type *expected_attach_type) 3177 { 3178 return libbpf_prog_type_by_name(prog->section_name, prog_type, 3179 expected_attach_type); 3180 } 3181 3182 int bpf_map__fd(struct bpf_map *map) 3183 { 3184 return map ? map->fd : -EINVAL; 3185 } 3186 3187 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 3188 { 3189 return map ? &map->def : ERR_PTR(-EINVAL); 3190 } 3191 3192 const char *bpf_map__name(struct bpf_map *map) 3193 { 3194 return map ? map->name : NULL; 3195 } 3196 3197 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 3198 { 3199 return map ? map->btf_key_type_id : 0; 3200 } 3201 3202 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 3203 { 3204 return map ? map->btf_value_type_id : 0; 3205 } 3206 3207 int bpf_map__set_priv(struct bpf_map *map, void *priv, 3208 bpf_map_clear_priv_t clear_priv) 3209 { 3210 if (!map) 3211 return -EINVAL; 3212 3213 if (map->priv) { 3214 if (map->clear_priv) 3215 map->clear_priv(map, map->priv); 3216 } 3217 3218 map->priv = priv; 3219 map->clear_priv = clear_priv; 3220 return 0; 3221 } 3222 3223 void *bpf_map__priv(struct bpf_map *map) 3224 { 3225 return map ? map->priv : ERR_PTR(-EINVAL); 3226 } 3227 3228 bool bpf_map__is_offload_neutral(struct bpf_map *map) 3229 { 3230 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 3231 } 3232 3233 bool bpf_map__is_internal(struct bpf_map *map) 3234 { 3235 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 3236 } 3237 3238 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 3239 { 3240 map->map_ifindex = ifindex; 3241 } 3242 3243 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 3244 { 3245 if (!bpf_map_type__is_map_in_map(map->def.type)) { 3246 pr_warning("error: unsupported map type\n"); 3247 return -EINVAL; 3248 } 3249 if (map->inner_map_fd != -1) { 3250 pr_warning("error: inner_map_fd already specified\n"); 3251 return -EINVAL; 3252 } 3253 map->inner_map_fd = fd; 3254 return 0; 3255 } 3256 3257 static struct bpf_map * 3258 __bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i) 3259 { 3260 ssize_t idx; 3261 struct bpf_map *s, *e; 3262 3263 if (!obj || !obj->maps) 3264 return NULL; 3265 3266 s = obj->maps; 3267 e = obj->maps + obj->nr_maps; 3268 3269 if ((m < s) || (m >= e)) { 3270 pr_warning("error in %s: map handler doesn't belong to object\n", 3271 __func__); 3272 return NULL; 3273 } 3274 3275 idx = (m - obj->maps) + i; 3276 if (idx >= obj->nr_maps || idx < 0) 3277 return NULL; 3278 return &obj->maps[idx]; 3279 } 3280 3281 struct bpf_map * 3282 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 3283 { 3284 if (prev == NULL) 3285 return obj->maps; 3286 3287 return __bpf_map__iter(prev, obj, 1); 3288 } 3289 3290 struct bpf_map * 3291 bpf_map__prev(struct bpf_map *next, struct bpf_object *obj) 3292 { 3293 if (next == NULL) { 3294 if (!obj->nr_maps) 3295 return NULL; 3296 return obj->maps + obj->nr_maps - 1; 3297 } 3298 3299 return __bpf_map__iter(next, obj, -1); 3300 } 3301 3302 struct bpf_map * 3303 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 3304 { 3305 struct bpf_map *pos; 3306 3307 bpf_object__for_each_map(pos, obj) { 3308 if (pos->name && !strcmp(pos->name, name)) 3309 return pos; 3310 } 3311 return NULL; 3312 } 3313 3314 int 3315 bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name) 3316 { 3317 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 3318 } 3319 3320 struct bpf_map * 3321 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 3322 { 3323 int i; 3324 3325 for (i = 0; i < obj->nr_maps; i++) { 3326 if (obj->maps[i].offset == offset) 3327 return &obj->maps[i]; 3328 } 3329 return ERR_PTR(-ENOENT); 3330 } 3331 3332 long libbpf_get_error(const void *ptr) 3333 { 3334 if (IS_ERR(ptr)) 3335 return PTR_ERR(ptr); 3336 return 0; 3337 } 3338 3339 int bpf_prog_load(const char *file, enum bpf_prog_type type, 3340 struct bpf_object **pobj, int *prog_fd) 3341 { 3342 struct bpf_prog_load_attr attr; 3343 3344 memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); 3345 attr.file = file; 3346 attr.prog_type = type; 3347 attr.expected_attach_type = 0; 3348 3349 return bpf_prog_load_xattr(&attr, pobj, prog_fd); 3350 } 3351 3352 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, 3353 struct bpf_object **pobj, int *prog_fd) 3354 { 3355 struct bpf_object_open_attr open_attr = { 3356 .file = attr->file, 3357 .prog_type = attr->prog_type, 3358 }; 3359 struct bpf_program *prog, *first_prog = NULL; 3360 enum bpf_attach_type expected_attach_type; 3361 enum bpf_prog_type prog_type; 3362 struct bpf_object *obj; 3363 struct bpf_map *map; 3364 int err; 3365 3366 if (!attr) 3367 return -EINVAL; 3368 if (!attr->file) 3369 return -EINVAL; 3370 3371 obj = bpf_object__open_xattr(&open_attr); 3372 if (IS_ERR_OR_NULL(obj)) 3373 return -ENOENT; 3374 3375 bpf_object__for_each_program(prog, obj) { 3376 /* 3377 * If type is not specified, try to guess it based on 3378 * section name. 3379 */ 3380 prog_type = attr->prog_type; 3381 prog->prog_ifindex = attr->ifindex; 3382 expected_attach_type = attr->expected_attach_type; 3383 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 3384 err = bpf_program__identify_section(prog, &prog_type, 3385 &expected_attach_type); 3386 if (err < 0) { 3387 bpf_object__close(obj); 3388 return -EINVAL; 3389 } 3390 } 3391 3392 bpf_program__set_type(prog, prog_type); 3393 bpf_program__set_expected_attach_type(prog, 3394 expected_attach_type); 3395 3396 prog->log_level = attr->log_level; 3397 if (!first_prog) 3398 first_prog = prog; 3399 } 3400 3401 bpf_object__for_each_map(map, obj) { 3402 if (!bpf_map__is_offload_neutral(map)) 3403 map->map_ifindex = attr->ifindex; 3404 } 3405 3406 if (!first_prog) { 3407 pr_warning("object file doesn't contain bpf program\n"); 3408 bpf_object__close(obj); 3409 return -ENOENT; 3410 } 3411 3412 err = bpf_object__load(obj); 3413 if (err) { 3414 bpf_object__close(obj); 3415 return -EINVAL; 3416 } 3417 3418 *pobj = obj; 3419 *prog_fd = bpf_program__fd(first_prog); 3420 return 0; 3421 } 3422 3423 enum bpf_perf_event_ret 3424 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 3425 void **copy_mem, size_t *copy_size, 3426 bpf_perf_event_print_t fn, void *private_data) 3427 { 3428 struct perf_event_mmap_page *header = mmap_mem; 3429 __u64 data_head = ring_buffer_read_head(header); 3430 __u64 data_tail = header->data_tail; 3431 void *base = ((__u8 *)header) + page_size; 3432 int ret = LIBBPF_PERF_EVENT_CONT; 3433 struct perf_event_header *ehdr; 3434 size_t ehdr_size; 3435 3436 while (data_head != data_tail) { 3437 ehdr = base + (data_tail & (mmap_size - 1)); 3438 ehdr_size = ehdr->size; 3439 3440 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 3441 void *copy_start = ehdr; 3442 size_t len_first = base + mmap_size - copy_start; 3443 size_t len_secnd = ehdr_size - len_first; 3444 3445 if (*copy_size < ehdr_size) { 3446 free(*copy_mem); 3447 *copy_mem = malloc(ehdr_size); 3448 if (!*copy_mem) { 3449 *copy_size = 0; 3450 ret = LIBBPF_PERF_EVENT_ERROR; 3451 break; 3452 } 3453 *copy_size = ehdr_size; 3454 } 3455 3456 memcpy(*copy_mem, copy_start, len_first); 3457 memcpy(*copy_mem + len_first, base, len_secnd); 3458 ehdr = *copy_mem; 3459 } 3460 3461 ret = fn(ehdr, private_data); 3462 data_tail += ehdr_size; 3463 if (ret != LIBBPF_PERF_EVENT_CONT) 3464 break; 3465 } 3466 3467 ring_buffer_write_tail(header, data_tail); 3468 return ret; 3469 } 3470 3471 struct bpf_prog_info_array_desc { 3472 int array_offset; /* e.g. offset of jited_prog_insns */ 3473 int count_offset; /* e.g. offset of jited_prog_len */ 3474 int size_offset; /* > 0: offset of rec size, 3475 * < 0: fix size of -size_offset 3476 */ 3477 }; 3478 3479 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { 3480 [BPF_PROG_INFO_JITED_INSNS] = { 3481 offsetof(struct bpf_prog_info, jited_prog_insns), 3482 offsetof(struct bpf_prog_info, jited_prog_len), 3483 -1, 3484 }, 3485 [BPF_PROG_INFO_XLATED_INSNS] = { 3486 offsetof(struct bpf_prog_info, xlated_prog_insns), 3487 offsetof(struct bpf_prog_info, xlated_prog_len), 3488 -1, 3489 }, 3490 [BPF_PROG_INFO_MAP_IDS] = { 3491 offsetof(struct bpf_prog_info, map_ids), 3492 offsetof(struct bpf_prog_info, nr_map_ids), 3493 -(int)sizeof(__u32), 3494 }, 3495 [BPF_PROG_INFO_JITED_KSYMS] = { 3496 offsetof(struct bpf_prog_info, jited_ksyms), 3497 offsetof(struct bpf_prog_info, nr_jited_ksyms), 3498 -(int)sizeof(__u64), 3499 }, 3500 [BPF_PROG_INFO_JITED_FUNC_LENS] = { 3501 offsetof(struct bpf_prog_info, jited_func_lens), 3502 offsetof(struct bpf_prog_info, nr_jited_func_lens), 3503 -(int)sizeof(__u32), 3504 }, 3505 [BPF_PROG_INFO_FUNC_INFO] = { 3506 offsetof(struct bpf_prog_info, func_info), 3507 offsetof(struct bpf_prog_info, nr_func_info), 3508 offsetof(struct bpf_prog_info, func_info_rec_size), 3509 }, 3510 [BPF_PROG_INFO_LINE_INFO] = { 3511 offsetof(struct bpf_prog_info, line_info), 3512 offsetof(struct bpf_prog_info, nr_line_info), 3513 offsetof(struct bpf_prog_info, line_info_rec_size), 3514 }, 3515 [BPF_PROG_INFO_JITED_LINE_INFO] = { 3516 offsetof(struct bpf_prog_info, jited_line_info), 3517 offsetof(struct bpf_prog_info, nr_jited_line_info), 3518 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 3519 }, 3520 [BPF_PROG_INFO_PROG_TAGS] = { 3521 offsetof(struct bpf_prog_info, prog_tags), 3522 offsetof(struct bpf_prog_info, nr_prog_tags), 3523 -(int)sizeof(__u8) * BPF_TAG_SIZE, 3524 }, 3525 3526 }; 3527 3528 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) 3529 { 3530 __u32 *array = (__u32 *)info; 3531 3532 if (offset >= 0) 3533 return array[offset / sizeof(__u32)]; 3534 return -(int)offset; 3535 } 3536 3537 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) 3538 { 3539 __u64 *array = (__u64 *)info; 3540 3541 if (offset >= 0) 3542 return array[offset / sizeof(__u64)]; 3543 return -(int)offset; 3544 } 3545 3546 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 3547 __u32 val) 3548 { 3549 __u32 *array = (__u32 *)info; 3550 3551 if (offset >= 0) 3552 array[offset / sizeof(__u32)] = val; 3553 } 3554 3555 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 3556 __u64 val) 3557 { 3558 __u64 *array = (__u64 *)info; 3559 3560 if (offset >= 0) 3561 array[offset / sizeof(__u64)] = val; 3562 } 3563 3564 struct bpf_prog_info_linear * 3565 bpf_program__get_prog_info_linear(int fd, __u64 arrays) 3566 { 3567 struct bpf_prog_info_linear *info_linear; 3568 struct bpf_prog_info info = {}; 3569 __u32 info_len = sizeof(info); 3570 __u32 data_len = 0; 3571 int i, err; 3572 void *ptr; 3573 3574 if (arrays >> BPF_PROG_INFO_LAST_ARRAY) 3575 return ERR_PTR(-EINVAL); 3576 3577 /* step 1: get array dimensions */ 3578 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 3579 if (err) { 3580 pr_debug("can't get prog info: %s", strerror(errno)); 3581 return ERR_PTR(-EFAULT); 3582 } 3583 3584 /* step 2: calculate total size of all arrays */ 3585 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3586 bool include_array = (arrays & (1UL << i)) > 0; 3587 struct bpf_prog_info_array_desc *desc; 3588 __u32 count, size; 3589 3590 desc = bpf_prog_info_array_desc + i; 3591 3592 /* kernel is too old to support this field */ 3593 if (info_len < desc->array_offset + sizeof(__u32) || 3594 info_len < desc->count_offset + sizeof(__u32) || 3595 (desc->size_offset > 0 && info_len < desc->size_offset)) 3596 include_array = false; 3597 3598 if (!include_array) { 3599 arrays &= ~(1UL << i); /* clear the bit */ 3600 continue; 3601 } 3602 3603 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3604 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3605 3606 data_len += count * size; 3607 } 3608 3609 /* step 3: allocate continuous memory */ 3610 data_len = roundup(data_len, sizeof(__u64)); 3611 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); 3612 if (!info_linear) 3613 return ERR_PTR(-ENOMEM); 3614 3615 /* step 4: fill data to info_linear->info */ 3616 info_linear->arrays = arrays; 3617 memset(&info_linear->info, 0, sizeof(info)); 3618 ptr = info_linear->data; 3619 3620 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3621 struct bpf_prog_info_array_desc *desc; 3622 __u32 count, size; 3623 3624 if ((arrays & (1UL << i)) == 0) 3625 continue; 3626 3627 desc = bpf_prog_info_array_desc + i; 3628 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3629 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3630 bpf_prog_info_set_offset_u32(&info_linear->info, 3631 desc->count_offset, count); 3632 bpf_prog_info_set_offset_u32(&info_linear->info, 3633 desc->size_offset, size); 3634 bpf_prog_info_set_offset_u64(&info_linear->info, 3635 desc->array_offset, 3636 ptr_to_u64(ptr)); 3637 ptr += count * size; 3638 } 3639 3640 /* step 5: call syscall again to get required arrays */ 3641 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 3642 if (err) { 3643 pr_debug("can't get prog info: %s", strerror(errno)); 3644 free(info_linear); 3645 return ERR_PTR(-EFAULT); 3646 } 3647 3648 /* step 6: verify the data */ 3649 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3650 struct bpf_prog_info_array_desc *desc; 3651 __u32 v1, v2; 3652 3653 if ((arrays & (1UL << i)) == 0) 3654 continue; 3655 3656 desc = bpf_prog_info_array_desc + i; 3657 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 3658 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3659 desc->count_offset); 3660 if (v1 != v2) 3661 pr_warning("%s: mismatch in element count\n", __func__); 3662 3663 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 3664 v2 = bpf_prog_info_read_offset_u32(&info_linear->info, 3665 desc->size_offset); 3666 if (v1 != v2) 3667 pr_warning("%s: mismatch in rec size\n", __func__); 3668 } 3669 3670 /* step 7: update info_len and data_len */ 3671 info_linear->info_len = sizeof(struct bpf_prog_info); 3672 info_linear->data_len = data_len; 3673 3674 return info_linear; 3675 } 3676 3677 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) 3678 { 3679 int i; 3680 3681 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3682 struct bpf_prog_info_array_desc *desc; 3683 __u64 addr, offs; 3684 3685 if ((info_linear->arrays & (1UL << i)) == 0) 3686 continue; 3687 3688 desc = bpf_prog_info_array_desc + i; 3689 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 3690 desc->array_offset); 3691 offs = addr - ptr_to_u64(info_linear->data); 3692 bpf_prog_info_set_offset_u64(&info_linear->info, 3693 desc->array_offset, offs); 3694 } 3695 } 3696 3697 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) 3698 { 3699 int i; 3700 3701 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { 3702 struct bpf_prog_info_array_desc *desc; 3703 __u64 addr, offs; 3704 3705 if ((info_linear->arrays & (1UL << i)) == 0) 3706 continue; 3707 3708 desc = bpf_prog_info_array_desc + i; 3709 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 3710 desc->array_offset); 3711 addr = offs + ptr_to_u64(info_linear->data); 3712 bpf_prog_info_set_offset_u64(&info_linear->info, 3713 desc->array_offset, addr); 3714 } 3715 } 3716