1 // SPDX-License-Identifier: LGPL-2.1 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * 11 * This program is free software; you can redistribute it and/or 12 * modify it under the terms of the GNU Lesser General Public 13 * License as published by the Free Software Foundation; 14 * version 2.1 of the License (not later!) 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU Lesser General Public License for more details. 20 * 21 * You should have received a copy of the GNU Lesser General Public 22 * License along with this program; if not, see <http://www.gnu.org/licenses> 23 */ 24 25 #include <stdlib.h> 26 #include <stdio.h> 27 #include <stdarg.h> 28 #include <libgen.h> 29 #include <inttypes.h> 30 #include <string.h> 31 #include <unistd.h> 32 #include <fcntl.h> 33 #include <errno.h> 34 #include <asm/unistd.h> 35 #include <linux/err.h> 36 #include <linux/kernel.h> 37 #include <linux/bpf.h> 38 #include <linux/list.h> 39 #include <linux/limits.h> 40 #include <sys/stat.h> 41 #include <sys/types.h> 42 #include <sys/vfs.h> 43 #include <libelf.h> 44 #include <gelf.h> 45 46 #include "libbpf.h" 47 #include "bpf.h" 48 49 #ifndef EM_BPF 50 #define EM_BPF 247 51 #endif 52 53 #ifndef BPF_FS_MAGIC 54 #define BPF_FS_MAGIC 0xcafe4a11 55 #endif 56 57 #define __printf(a, b) __attribute__((format(printf, a, b))) 58 59 __printf(1, 2) 60 static int __base_pr(const char *format, ...) 61 { 62 va_list args; 63 int err; 64 65 va_start(args, format); 66 err = vfprintf(stderr, format, args); 67 va_end(args); 68 return err; 69 } 70 71 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; 72 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; 73 static __printf(1, 2) libbpf_print_fn_t __pr_debug; 74 75 #define __pr(func, fmt, ...) \ 76 do { \ 77 if ((func)) \ 78 (func)("libbpf: " fmt, ##__VA_ARGS__); \ 79 } while (0) 80 81 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) 82 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) 83 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) 84 85 void libbpf_set_print(libbpf_print_fn_t warn, 86 libbpf_print_fn_t info, 87 libbpf_print_fn_t debug) 88 { 89 __pr_warning = warn; 90 __pr_info = info; 91 __pr_debug = debug; 92 } 93 94 #define STRERR_BUFSIZE 128 95 96 #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) 97 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) 98 #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) 99 100 static const char *libbpf_strerror_table[NR_ERRNO] = { 101 [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", 102 [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", 103 [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", 104 [ERRCODE_OFFSET(ENDIAN)] = "Endian mismatch", 105 [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", 106 [ERRCODE_OFFSET(RELOC)] = "Relocation failed", 107 [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", 108 [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", 109 [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", 110 [ERRCODE_OFFSET(PROGTYPE)] = "Kernel doesn't support this program type", 111 [ERRCODE_OFFSET(WRNGPID)] = "Wrong pid in netlink message", 112 [ERRCODE_OFFSET(INVSEQ)] = "Invalid netlink sequence", 113 }; 114 115 int libbpf_strerror(int err, char *buf, size_t size) 116 { 117 if (!buf || !size) 118 return -1; 119 120 err = err > 0 ? err : -err; 121 122 if (err < __LIBBPF_ERRNO__START) { 123 int ret; 124 125 ret = strerror_r(err, buf, size); 126 buf[size - 1] = '\0'; 127 return ret; 128 } 129 130 if (err < __LIBBPF_ERRNO__END) { 131 const char *msg; 132 133 msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; 134 snprintf(buf, size, "%s", msg); 135 buf[size - 1] = '\0'; 136 return 0; 137 } 138 139 snprintf(buf, size, "Unknown libbpf error %d", err); 140 buf[size - 1] = '\0'; 141 return -1; 142 } 143 144 #define CHECK_ERR(action, err, out) do { \ 145 err = action; \ 146 if (err) \ 147 goto out; \ 148 } while(0) 149 150 151 /* Copied from tools/perf/util/util.h */ 152 #ifndef zfree 153 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 154 #endif 155 156 #ifndef zclose 157 # define zclose(fd) ({ \ 158 int ___err = 0; \ 159 if ((fd) >= 0) \ 160 ___err = close((fd)); \ 161 fd = -1; \ 162 ___err; }) 163 #endif 164 165 #ifdef HAVE_LIBELF_MMAP_SUPPORT 166 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 167 #else 168 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 169 #endif 170 171 /* 172 * bpf_prog should be a better name but it has been used in 173 * linux/filter.h. 174 */ 175 struct bpf_program { 176 /* Index in elf obj file, for relocation use. */ 177 int idx; 178 char *name; 179 char *section_name; 180 struct bpf_insn *insns; 181 size_t insns_cnt, main_prog_cnt; 182 enum bpf_prog_type type; 183 184 struct reloc_desc { 185 enum { 186 RELO_LD64, 187 RELO_CALL, 188 } type; 189 int insn_idx; 190 union { 191 int map_idx; 192 int text_off; 193 }; 194 } *reloc_desc; 195 int nr_reloc; 196 197 struct { 198 int nr; 199 int *fds; 200 } instances; 201 bpf_program_prep_t preprocessor; 202 203 struct bpf_object *obj; 204 void *priv; 205 bpf_program_clear_priv_t clear_priv; 206 }; 207 208 struct bpf_map { 209 int fd; 210 char *name; 211 size_t offset; 212 struct bpf_map_def def; 213 void *priv; 214 bpf_map_clear_priv_t clear_priv; 215 }; 216 217 static LIST_HEAD(bpf_objects_list); 218 219 struct bpf_object { 220 char license[64]; 221 u32 kern_version; 222 223 struct bpf_program *programs; 224 size_t nr_programs; 225 struct bpf_map *maps; 226 size_t nr_maps; 227 228 bool loaded; 229 230 /* 231 * Information when doing elf related work. Only valid if fd 232 * is valid. 233 */ 234 struct { 235 int fd; 236 void *obj_buf; 237 size_t obj_buf_sz; 238 Elf *elf; 239 GElf_Ehdr ehdr; 240 Elf_Data *symbols; 241 size_t strtabidx; 242 struct { 243 GElf_Shdr shdr; 244 Elf_Data *data; 245 } *reloc; 246 int nr_reloc; 247 int maps_shndx; 248 int text_shndx; 249 } efile; 250 /* 251 * All loaded bpf_object is linked in a list, which is 252 * hidden to caller. bpf_objects__<func> handlers deal with 253 * all objects. 254 */ 255 struct list_head list; 256 257 void *priv; 258 bpf_object_clear_priv_t clear_priv; 259 260 char path[]; 261 }; 262 #define obj_elf_valid(o) ((o)->efile.elf) 263 264 static void bpf_program__unload(struct bpf_program *prog) 265 { 266 int i; 267 268 if (!prog) 269 return; 270 271 /* 272 * If the object is opened but the program was never loaded, 273 * it is possible that prog->instances.nr == -1. 274 */ 275 if (prog->instances.nr > 0) { 276 for (i = 0; i < prog->instances.nr; i++) 277 zclose(prog->instances.fds[i]); 278 } else if (prog->instances.nr != -1) { 279 pr_warning("Internal error: instances.nr is %d\n", 280 prog->instances.nr); 281 } 282 283 prog->instances.nr = -1; 284 zfree(&prog->instances.fds); 285 } 286 287 static void bpf_program__exit(struct bpf_program *prog) 288 { 289 if (!prog) 290 return; 291 292 if (prog->clear_priv) 293 prog->clear_priv(prog, prog->priv); 294 295 prog->priv = NULL; 296 prog->clear_priv = NULL; 297 298 bpf_program__unload(prog); 299 zfree(&prog->name); 300 zfree(&prog->section_name); 301 zfree(&prog->insns); 302 zfree(&prog->reloc_desc); 303 304 prog->nr_reloc = 0; 305 prog->insns_cnt = 0; 306 prog->idx = -1; 307 } 308 309 static int 310 bpf_program__init(void *data, size_t size, char *section_name, int idx, 311 struct bpf_program *prog) 312 { 313 if (size < sizeof(struct bpf_insn)) { 314 pr_warning("corrupted section '%s'\n", section_name); 315 return -EINVAL; 316 } 317 318 bzero(prog, sizeof(*prog)); 319 320 prog->section_name = strdup(section_name); 321 if (!prog->section_name) { 322 pr_warning("failed to alloc name for prog under section(%d) %s\n", 323 idx, section_name); 324 goto errout; 325 } 326 327 prog->insns = malloc(size); 328 if (!prog->insns) { 329 pr_warning("failed to alloc insns for prog under section %s\n", 330 section_name); 331 goto errout; 332 } 333 prog->insns_cnt = size / sizeof(struct bpf_insn); 334 memcpy(prog->insns, data, 335 prog->insns_cnt * sizeof(struct bpf_insn)); 336 prog->idx = idx; 337 prog->instances.fds = NULL; 338 prog->instances.nr = -1; 339 prog->type = BPF_PROG_TYPE_KPROBE; 340 341 return 0; 342 errout: 343 bpf_program__exit(prog); 344 return -ENOMEM; 345 } 346 347 static int 348 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 349 char *section_name, int idx) 350 { 351 struct bpf_program prog, *progs; 352 int nr_progs, err; 353 354 err = bpf_program__init(data, size, section_name, idx, &prog); 355 if (err) 356 return err; 357 358 progs = obj->programs; 359 nr_progs = obj->nr_programs; 360 361 progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); 362 if (!progs) { 363 /* 364 * In this case the original obj->programs 365 * is still valid, so don't need special treat for 366 * bpf_close_object(). 367 */ 368 pr_warning("failed to alloc a new program under section '%s'\n", 369 section_name); 370 bpf_program__exit(&prog); 371 return -ENOMEM; 372 } 373 374 pr_debug("found program %s\n", prog.section_name); 375 obj->programs = progs; 376 obj->nr_programs = nr_progs + 1; 377 prog.obj = obj; 378 progs[nr_progs] = prog; 379 return 0; 380 } 381 382 static int 383 bpf_object__init_prog_names(struct bpf_object *obj) 384 { 385 Elf_Data *symbols = obj->efile.symbols; 386 struct bpf_program *prog; 387 size_t pi, si; 388 389 for (pi = 0; pi < obj->nr_programs; pi++) { 390 const char *name = NULL; 391 392 prog = &obj->programs[pi]; 393 if (prog->idx == obj->efile.text_shndx) { 394 name = ".text"; 395 goto skip_search; 396 } 397 398 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 399 si++) { 400 GElf_Sym sym; 401 402 if (!gelf_getsym(symbols, si, &sym)) 403 continue; 404 if (sym.st_shndx != prog->idx) 405 continue; 406 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 407 continue; 408 409 name = elf_strptr(obj->efile.elf, 410 obj->efile.strtabidx, 411 sym.st_name); 412 if (!name) { 413 pr_warning("failed to get sym name string for prog %s\n", 414 prog->section_name); 415 return -LIBBPF_ERRNO__LIBELF; 416 } 417 } 418 419 if (!name) { 420 pr_warning("failed to find sym for prog %s\n", 421 prog->section_name); 422 return -EINVAL; 423 } 424 skip_search: 425 prog->name = strdup(name); 426 if (!prog->name) { 427 pr_warning("failed to allocate memory for prog sym %s\n", 428 name); 429 return -ENOMEM; 430 } 431 } 432 433 return 0; 434 } 435 436 static struct bpf_object *bpf_object__new(const char *path, 437 void *obj_buf, 438 size_t obj_buf_sz) 439 { 440 struct bpf_object *obj; 441 442 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 443 if (!obj) { 444 pr_warning("alloc memory failed for %s\n", path); 445 return ERR_PTR(-ENOMEM); 446 } 447 448 strcpy(obj->path, path); 449 obj->efile.fd = -1; 450 451 /* 452 * Caller of this function should also calls 453 * bpf_object__elf_finish() after data collection to return 454 * obj_buf to user. If not, we should duplicate the buffer to 455 * avoid user freeing them before elf finish. 456 */ 457 obj->efile.obj_buf = obj_buf; 458 obj->efile.obj_buf_sz = obj_buf_sz; 459 obj->efile.maps_shndx = -1; 460 461 obj->loaded = false; 462 463 INIT_LIST_HEAD(&obj->list); 464 list_add(&obj->list, &bpf_objects_list); 465 return obj; 466 } 467 468 static void bpf_object__elf_finish(struct bpf_object *obj) 469 { 470 if (!obj_elf_valid(obj)) 471 return; 472 473 if (obj->efile.elf) { 474 elf_end(obj->efile.elf); 475 obj->efile.elf = NULL; 476 } 477 obj->efile.symbols = NULL; 478 479 zfree(&obj->efile.reloc); 480 obj->efile.nr_reloc = 0; 481 zclose(obj->efile.fd); 482 obj->efile.obj_buf = NULL; 483 obj->efile.obj_buf_sz = 0; 484 } 485 486 static int bpf_object__elf_init(struct bpf_object *obj) 487 { 488 int err = 0; 489 GElf_Ehdr *ep; 490 491 if (obj_elf_valid(obj)) { 492 pr_warning("elf init: internal error\n"); 493 return -LIBBPF_ERRNO__LIBELF; 494 } 495 496 if (obj->efile.obj_buf_sz > 0) { 497 /* 498 * obj_buf should have been validated by 499 * bpf_object__open_buffer(). 500 */ 501 obj->efile.elf = elf_memory(obj->efile.obj_buf, 502 obj->efile.obj_buf_sz); 503 } else { 504 obj->efile.fd = open(obj->path, O_RDONLY); 505 if (obj->efile.fd < 0) { 506 pr_warning("failed to open %s: %s\n", obj->path, 507 strerror(errno)); 508 return -errno; 509 } 510 511 obj->efile.elf = elf_begin(obj->efile.fd, 512 LIBBPF_ELF_C_READ_MMAP, 513 NULL); 514 } 515 516 if (!obj->efile.elf) { 517 pr_warning("failed to open %s as ELF file\n", 518 obj->path); 519 err = -LIBBPF_ERRNO__LIBELF; 520 goto errout; 521 } 522 523 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 524 pr_warning("failed to get EHDR from %s\n", 525 obj->path); 526 err = -LIBBPF_ERRNO__FORMAT; 527 goto errout; 528 } 529 ep = &obj->efile.ehdr; 530 531 /* Old LLVM set e_machine to EM_NONE */ 532 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { 533 pr_warning("%s is not an eBPF object file\n", 534 obj->path); 535 err = -LIBBPF_ERRNO__FORMAT; 536 goto errout; 537 } 538 539 return 0; 540 errout: 541 bpf_object__elf_finish(obj); 542 return err; 543 } 544 545 static int 546 bpf_object__check_endianness(struct bpf_object *obj) 547 { 548 static unsigned int const endian = 1; 549 550 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 551 case ELFDATA2LSB: 552 /* We are big endian, BPF obj is little endian. */ 553 if (*(unsigned char const *)&endian != 1) 554 goto mismatch; 555 break; 556 557 case ELFDATA2MSB: 558 /* We are little endian, BPF obj is big endian. */ 559 if (*(unsigned char const *)&endian != 0) 560 goto mismatch; 561 break; 562 default: 563 return -LIBBPF_ERRNO__ENDIAN; 564 } 565 566 return 0; 567 568 mismatch: 569 pr_warning("Error: endianness mismatch.\n"); 570 return -LIBBPF_ERRNO__ENDIAN; 571 } 572 573 static int 574 bpf_object__init_license(struct bpf_object *obj, 575 void *data, size_t size) 576 { 577 memcpy(obj->license, data, 578 min(size, sizeof(obj->license) - 1)); 579 pr_debug("license of %s is %s\n", obj->path, obj->license); 580 return 0; 581 } 582 583 static int 584 bpf_object__init_kversion(struct bpf_object *obj, 585 void *data, size_t size) 586 { 587 u32 kver; 588 589 if (size != sizeof(kver)) { 590 pr_warning("invalid kver section in %s\n", obj->path); 591 return -LIBBPF_ERRNO__FORMAT; 592 } 593 memcpy(&kver, data, sizeof(kver)); 594 obj->kern_version = kver; 595 pr_debug("kernel version of %s is %x\n", obj->path, 596 obj->kern_version); 597 return 0; 598 } 599 600 static int compare_bpf_map(const void *_a, const void *_b) 601 { 602 const struct bpf_map *a = _a; 603 const struct bpf_map *b = _b; 604 605 return a->offset - b->offset; 606 } 607 608 static int 609 bpf_object__init_maps(struct bpf_object *obj) 610 { 611 int i, map_idx, map_def_sz, nr_maps = 0; 612 Elf_Scn *scn; 613 Elf_Data *data; 614 Elf_Data *symbols = obj->efile.symbols; 615 616 if (obj->efile.maps_shndx < 0) 617 return -EINVAL; 618 if (!symbols) 619 return -EINVAL; 620 621 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); 622 if (scn) 623 data = elf_getdata(scn, NULL); 624 if (!scn || !data) { 625 pr_warning("failed to get Elf_Data from map section %d\n", 626 obj->efile.maps_shndx); 627 return -EINVAL; 628 } 629 630 /* 631 * Count number of maps. Each map has a name. 632 * Array of maps is not supported: only the first element is 633 * considered. 634 * 635 * TODO: Detect array of map and report error. 636 */ 637 for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 638 GElf_Sym sym; 639 640 if (!gelf_getsym(symbols, i, &sym)) 641 continue; 642 if (sym.st_shndx != obj->efile.maps_shndx) 643 continue; 644 nr_maps++; 645 } 646 647 /* Alloc obj->maps and fill nr_maps. */ 648 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 649 nr_maps, data->d_size); 650 651 if (!nr_maps) 652 return 0; 653 654 /* Assume equally sized map definitions */ 655 map_def_sz = data->d_size / nr_maps; 656 if (!data->d_size || (data->d_size % nr_maps) != 0) { 657 pr_warning("unable to determine map definition size " 658 "section %s, %d maps in %zd bytes\n", 659 obj->path, nr_maps, data->d_size); 660 return -EINVAL; 661 } 662 663 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 664 if (!obj->maps) { 665 pr_warning("alloc maps for object failed\n"); 666 return -ENOMEM; 667 } 668 obj->nr_maps = nr_maps; 669 670 /* 671 * fill all fd with -1 so won't close incorrect 672 * fd (fd=0 is stdin) when failure (zclose won't close 673 * negative fd)). 674 */ 675 for (i = 0; i < nr_maps; i++) 676 obj->maps[i].fd = -1; 677 678 /* 679 * Fill obj->maps using data in "maps" section. 680 */ 681 for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 682 GElf_Sym sym; 683 const char *map_name; 684 struct bpf_map_def *def; 685 686 if (!gelf_getsym(symbols, i, &sym)) 687 continue; 688 if (sym.st_shndx != obj->efile.maps_shndx) 689 continue; 690 691 map_name = elf_strptr(obj->efile.elf, 692 obj->efile.strtabidx, 693 sym.st_name); 694 obj->maps[map_idx].offset = sym.st_value; 695 if (sym.st_value + map_def_sz > data->d_size) { 696 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 697 obj->path, map_name); 698 return -EINVAL; 699 } 700 701 obj->maps[map_idx].name = strdup(map_name); 702 if (!obj->maps[map_idx].name) { 703 pr_warning("failed to alloc map name\n"); 704 return -ENOMEM; 705 } 706 pr_debug("map %d is \"%s\"\n", map_idx, 707 obj->maps[map_idx].name); 708 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 709 /* 710 * If the definition of the map in the object file fits in 711 * bpf_map_def, copy it. Any extra fields in our version 712 * of bpf_map_def will default to zero as a result of the 713 * calloc above. 714 */ 715 if (map_def_sz <= sizeof(struct bpf_map_def)) { 716 memcpy(&obj->maps[map_idx].def, def, map_def_sz); 717 } else { 718 /* 719 * Here the map structure being read is bigger than what 720 * we expect, truncate if the excess bits are all zero. 721 * If they are not zero, reject this map as 722 * incompatible. 723 */ 724 char *b; 725 for (b = ((char *)def) + sizeof(struct bpf_map_def); 726 b < ((char *)def) + map_def_sz; b++) { 727 if (*b != 0) { 728 pr_warning("maps section in %s: \"%s\" " 729 "has unrecognized, non-zero " 730 "options\n", 731 obj->path, map_name); 732 return -EINVAL; 733 } 734 } 735 memcpy(&obj->maps[map_idx].def, def, 736 sizeof(struct bpf_map_def)); 737 } 738 map_idx++; 739 } 740 741 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); 742 return 0; 743 } 744 745 static bool section_have_execinstr(struct bpf_object *obj, int idx) 746 { 747 Elf_Scn *scn; 748 GElf_Shdr sh; 749 750 scn = elf_getscn(obj->efile.elf, idx); 751 if (!scn) 752 return false; 753 754 if (gelf_getshdr(scn, &sh) != &sh) 755 return false; 756 757 if (sh.sh_flags & SHF_EXECINSTR) 758 return true; 759 760 return false; 761 } 762 763 static int bpf_object__elf_collect(struct bpf_object *obj) 764 { 765 Elf *elf = obj->efile.elf; 766 GElf_Ehdr *ep = &obj->efile.ehdr; 767 Elf_Scn *scn = NULL; 768 int idx = 0, err = 0; 769 770 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 771 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 772 pr_warning("failed to get e_shstrndx from %s\n", 773 obj->path); 774 return -LIBBPF_ERRNO__FORMAT; 775 } 776 777 while ((scn = elf_nextscn(elf, scn)) != NULL) { 778 char *name; 779 GElf_Shdr sh; 780 Elf_Data *data; 781 782 idx++; 783 if (gelf_getshdr(scn, &sh) != &sh) { 784 pr_warning("failed to get section(%d) header from %s\n", 785 idx, obj->path); 786 err = -LIBBPF_ERRNO__FORMAT; 787 goto out; 788 } 789 790 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 791 if (!name) { 792 pr_warning("failed to get section(%d) name from %s\n", 793 idx, obj->path); 794 err = -LIBBPF_ERRNO__FORMAT; 795 goto out; 796 } 797 798 data = elf_getdata(scn, 0); 799 if (!data) { 800 pr_warning("failed to get section(%d) data from %s(%s)\n", 801 idx, name, obj->path); 802 err = -LIBBPF_ERRNO__FORMAT; 803 goto out; 804 } 805 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 806 idx, name, (unsigned long)data->d_size, 807 (int)sh.sh_link, (unsigned long)sh.sh_flags, 808 (int)sh.sh_type); 809 810 if (strcmp(name, "license") == 0) 811 err = bpf_object__init_license(obj, 812 data->d_buf, 813 data->d_size); 814 else if (strcmp(name, "version") == 0) 815 err = bpf_object__init_kversion(obj, 816 data->d_buf, 817 data->d_size); 818 else if (strcmp(name, "maps") == 0) 819 obj->efile.maps_shndx = idx; 820 else if (sh.sh_type == SHT_SYMTAB) { 821 if (obj->efile.symbols) { 822 pr_warning("bpf: multiple SYMTAB in %s\n", 823 obj->path); 824 err = -LIBBPF_ERRNO__FORMAT; 825 } else { 826 obj->efile.symbols = data; 827 obj->efile.strtabidx = sh.sh_link; 828 } 829 } else if ((sh.sh_type == SHT_PROGBITS) && 830 (sh.sh_flags & SHF_EXECINSTR) && 831 (data->d_size > 0)) { 832 if (strcmp(name, ".text") == 0) 833 obj->efile.text_shndx = idx; 834 err = bpf_object__add_program(obj, data->d_buf, 835 data->d_size, name, idx); 836 if (err) { 837 char errmsg[STRERR_BUFSIZE]; 838 839 strerror_r(-err, errmsg, sizeof(errmsg)); 840 pr_warning("failed to alloc program %s (%s): %s", 841 name, obj->path, errmsg); 842 } 843 } else if (sh.sh_type == SHT_REL) { 844 void *reloc = obj->efile.reloc; 845 int nr_reloc = obj->efile.nr_reloc + 1; 846 int sec = sh.sh_info; /* points to other section */ 847 848 /* Only do relo for section with exec instructions */ 849 if (!section_have_execinstr(obj, sec)) { 850 pr_debug("skip relo %s(%d) for section(%d)\n", 851 name, idx, sec); 852 continue; 853 } 854 855 reloc = realloc(reloc, 856 sizeof(*obj->efile.reloc) * nr_reloc); 857 if (!reloc) { 858 pr_warning("realloc failed\n"); 859 err = -ENOMEM; 860 } else { 861 int n = nr_reloc - 1; 862 863 obj->efile.reloc = reloc; 864 obj->efile.nr_reloc = nr_reloc; 865 866 obj->efile.reloc[n].shdr = sh; 867 obj->efile.reloc[n].data = data; 868 } 869 } else { 870 pr_debug("skip section(%d) %s\n", idx, name); 871 } 872 if (err) 873 goto out; 874 } 875 876 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 877 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 878 return LIBBPF_ERRNO__FORMAT; 879 } 880 if (obj->efile.maps_shndx >= 0) { 881 err = bpf_object__init_maps(obj); 882 if (err) 883 goto out; 884 } 885 err = bpf_object__init_prog_names(obj); 886 out: 887 return err; 888 } 889 890 static struct bpf_program * 891 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 892 { 893 struct bpf_program *prog; 894 size_t i; 895 896 for (i = 0; i < obj->nr_programs; i++) { 897 prog = &obj->programs[i]; 898 if (prog->idx == idx) 899 return prog; 900 } 901 return NULL; 902 } 903 904 static int 905 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 906 Elf_Data *data, struct bpf_object *obj) 907 { 908 Elf_Data *symbols = obj->efile.symbols; 909 int text_shndx = obj->efile.text_shndx; 910 int maps_shndx = obj->efile.maps_shndx; 911 struct bpf_map *maps = obj->maps; 912 size_t nr_maps = obj->nr_maps; 913 int i, nrels; 914 915 pr_debug("collecting relocating info for: '%s'\n", 916 prog->section_name); 917 nrels = shdr->sh_size / shdr->sh_entsize; 918 919 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 920 if (!prog->reloc_desc) { 921 pr_warning("failed to alloc memory in relocation\n"); 922 return -ENOMEM; 923 } 924 prog->nr_reloc = nrels; 925 926 for (i = 0; i < nrels; i++) { 927 GElf_Sym sym; 928 GElf_Rel rel; 929 unsigned int insn_idx; 930 struct bpf_insn *insns = prog->insns; 931 size_t map_idx; 932 933 if (!gelf_getrel(data, i, &rel)) { 934 pr_warning("relocation: failed to get %d reloc\n", i); 935 return -LIBBPF_ERRNO__FORMAT; 936 } 937 938 if (!gelf_getsym(symbols, 939 GELF_R_SYM(rel.r_info), 940 &sym)) { 941 pr_warning("relocation: symbol %"PRIx64" not found\n", 942 GELF_R_SYM(rel.r_info)); 943 return -LIBBPF_ERRNO__FORMAT; 944 } 945 pr_debug("relo for %lld value %lld name %d\n", 946 (long long) (rel.r_info >> 32), 947 (long long) sym.st_value, sym.st_name); 948 949 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) { 950 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", 951 prog->section_name, sym.st_shndx); 952 return -LIBBPF_ERRNO__RELOC; 953 } 954 955 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 956 pr_debug("relocation: insn_idx=%u\n", insn_idx); 957 958 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 959 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 960 pr_warning("incorrect bpf_call opcode\n"); 961 return -LIBBPF_ERRNO__RELOC; 962 } 963 prog->reloc_desc[i].type = RELO_CALL; 964 prog->reloc_desc[i].insn_idx = insn_idx; 965 prog->reloc_desc[i].text_off = sym.st_value; 966 continue; 967 } 968 969 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 970 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 971 insn_idx, insns[insn_idx].code); 972 return -LIBBPF_ERRNO__RELOC; 973 } 974 975 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */ 976 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 977 if (maps[map_idx].offset == sym.st_value) { 978 pr_debug("relocation: find map %zd (%s) for insn %u\n", 979 map_idx, maps[map_idx].name, insn_idx); 980 break; 981 } 982 } 983 984 if (map_idx >= nr_maps) { 985 pr_warning("bpf relocation: map_idx %d large than %d\n", 986 (int)map_idx, (int)nr_maps - 1); 987 return -LIBBPF_ERRNO__RELOC; 988 } 989 990 prog->reloc_desc[i].type = RELO_LD64; 991 prog->reloc_desc[i].insn_idx = insn_idx; 992 prog->reloc_desc[i].map_idx = map_idx; 993 } 994 return 0; 995 } 996 997 static int 998 bpf_object__create_maps(struct bpf_object *obj) 999 { 1000 unsigned int i; 1001 1002 for (i = 0; i < obj->nr_maps; i++) { 1003 struct bpf_map_def *def = &obj->maps[i].def; 1004 int *pfd = &obj->maps[i].fd; 1005 1006 *pfd = bpf_create_map_name(def->type, 1007 obj->maps[i].name, 1008 def->key_size, 1009 def->value_size, 1010 def->max_entries, 1011 def->map_flags); 1012 if (*pfd < 0) { 1013 size_t j; 1014 int err = *pfd; 1015 1016 pr_warning("failed to create map (name: '%s'): %s\n", 1017 obj->maps[i].name, 1018 strerror(errno)); 1019 for (j = 0; j < i; j++) 1020 zclose(obj->maps[j].fd); 1021 return err; 1022 } 1023 pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd); 1024 } 1025 1026 return 0; 1027 } 1028 1029 static int 1030 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 1031 struct reloc_desc *relo) 1032 { 1033 struct bpf_insn *insn, *new_insn; 1034 struct bpf_program *text; 1035 size_t new_cnt; 1036 1037 if (relo->type != RELO_CALL) 1038 return -LIBBPF_ERRNO__RELOC; 1039 1040 if (prog->idx == obj->efile.text_shndx) { 1041 pr_warning("relo in .text insn %d into off %d\n", 1042 relo->insn_idx, relo->text_off); 1043 return -LIBBPF_ERRNO__RELOC; 1044 } 1045 1046 if (prog->main_prog_cnt == 0) { 1047 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 1048 if (!text) { 1049 pr_warning("no .text section found yet relo into text exist\n"); 1050 return -LIBBPF_ERRNO__RELOC; 1051 } 1052 new_cnt = prog->insns_cnt + text->insns_cnt; 1053 new_insn = realloc(prog->insns, new_cnt * sizeof(*insn)); 1054 if (!new_insn) { 1055 pr_warning("oom in prog realloc\n"); 1056 return -ENOMEM; 1057 } 1058 memcpy(new_insn + prog->insns_cnt, text->insns, 1059 text->insns_cnt * sizeof(*insn)); 1060 prog->insns = new_insn; 1061 prog->main_prog_cnt = prog->insns_cnt; 1062 prog->insns_cnt = new_cnt; 1063 } 1064 insn = &prog->insns[relo->insn_idx]; 1065 insn->imm += prog->main_prog_cnt - relo->insn_idx; 1066 pr_debug("added %zd insn from %s to prog %s\n", 1067 text->insns_cnt, text->section_name, prog->section_name); 1068 return 0; 1069 } 1070 1071 static int 1072 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1073 { 1074 int i, err; 1075 1076 if (!prog || !prog->reloc_desc) 1077 return 0; 1078 1079 for (i = 0; i < prog->nr_reloc; i++) { 1080 if (prog->reloc_desc[i].type == RELO_LD64) { 1081 struct bpf_insn *insns = prog->insns; 1082 int insn_idx, map_idx; 1083 1084 insn_idx = prog->reloc_desc[i].insn_idx; 1085 map_idx = prog->reloc_desc[i].map_idx; 1086 1087 if (insn_idx >= (int)prog->insns_cnt) { 1088 pr_warning("relocation out of range: '%s'\n", 1089 prog->section_name); 1090 return -LIBBPF_ERRNO__RELOC; 1091 } 1092 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1093 insns[insn_idx].imm = obj->maps[map_idx].fd; 1094 } else { 1095 err = bpf_program__reloc_text(prog, obj, 1096 &prog->reloc_desc[i]); 1097 if (err) 1098 return err; 1099 } 1100 } 1101 1102 zfree(&prog->reloc_desc); 1103 prog->nr_reloc = 0; 1104 return 0; 1105 } 1106 1107 1108 static int 1109 bpf_object__relocate(struct bpf_object *obj) 1110 { 1111 struct bpf_program *prog; 1112 size_t i; 1113 int err; 1114 1115 for (i = 0; i < obj->nr_programs; i++) { 1116 prog = &obj->programs[i]; 1117 1118 err = bpf_program__relocate(prog, obj); 1119 if (err) { 1120 pr_warning("failed to relocate '%s'\n", 1121 prog->section_name); 1122 return err; 1123 } 1124 } 1125 return 0; 1126 } 1127 1128 static int bpf_object__collect_reloc(struct bpf_object *obj) 1129 { 1130 int i, err; 1131 1132 if (!obj_elf_valid(obj)) { 1133 pr_warning("Internal error: elf object is closed\n"); 1134 return -LIBBPF_ERRNO__INTERNAL; 1135 } 1136 1137 for (i = 0; i < obj->efile.nr_reloc; i++) { 1138 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 1139 Elf_Data *data = obj->efile.reloc[i].data; 1140 int idx = shdr->sh_info; 1141 struct bpf_program *prog; 1142 1143 if (shdr->sh_type != SHT_REL) { 1144 pr_warning("internal error at %d\n", __LINE__); 1145 return -LIBBPF_ERRNO__INTERNAL; 1146 } 1147 1148 prog = bpf_object__find_prog_by_idx(obj, idx); 1149 if (!prog) { 1150 pr_warning("relocation failed: no section(%d)\n", idx); 1151 return -LIBBPF_ERRNO__RELOC; 1152 } 1153 1154 err = bpf_program__collect_reloc(prog, 1155 shdr, data, 1156 obj); 1157 if (err) 1158 return err; 1159 } 1160 return 0; 1161 } 1162 1163 static int 1164 load_program(enum bpf_prog_type type, const char *name, struct bpf_insn *insns, 1165 int insns_cnt, char *license, u32 kern_version, int *pfd) 1166 { 1167 int ret; 1168 char *log_buf; 1169 1170 if (!insns || !insns_cnt) 1171 return -EINVAL; 1172 1173 log_buf = malloc(BPF_LOG_BUF_SIZE); 1174 if (!log_buf) 1175 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 1176 1177 ret = bpf_load_program_name(type, name, insns, insns_cnt, license, 1178 kern_version, log_buf, BPF_LOG_BUF_SIZE); 1179 1180 if (ret >= 0) { 1181 *pfd = ret; 1182 ret = 0; 1183 goto out; 1184 } 1185 1186 ret = -LIBBPF_ERRNO__LOAD; 1187 pr_warning("load bpf program failed: %s\n", strerror(errno)); 1188 1189 if (log_buf && log_buf[0] != '\0') { 1190 ret = -LIBBPF_ERRNO__VERIFY; 1191 pr_warning("-- BEGIN DUMP LOG ---\n"); 1192 pr_warning("\n%s\n", log_buf); 1193 pr_warning("-- END LOG --\n"); 1194 } else if (insns_cnt >= BPF_MAXINSNS) { 1195 pr_warning("Program too large (%d insns), at most %d insns\n", 1196 insns_cnt, BPF_MAXINSNS); 1197 ret = -LIBBPF_ERRNO__PROG2BIG; 1198 } else { 1199 /* Wrong program type? */ 1200 if (type != BPF_PROG_TYPE_KPROBE) { 1201 int fd; 1202 1203 fd = bpf_load_program_name(BPF_PROG_TYPE_KPROBE, name, 1204 insns, insns_cnt, license, 1205 kern_version, NULL, 0); 1206 if (fd >= 0) { 1207 close(fd); 1208 ret = -LIBBPF_ERRNO__PROGTYPE; 1209 goto out; 1210 } 1211 } 1212 1213 if (log_buf) 1214 ret = -LIBBPF_ERRNO__KVER; 1215 } 1216 1217 out: 1218 free(log_buf); 1219 return ret; 1220 } 1221 1222 static int 1223 bpf_program__load(struct bpf_program *prog, 1224 char *license, u32 kern_version) 1225 { 1226 int err = 0, fd, i; 1227 1228 if (prog->instances.nr < 0 || !prog->instances.fds) { 1229 if (prog->preprocessor) { 1230 pr_warning("Internal error: can't load program '%s'\n", 1231 prog->section_name); 1232 return -LIBBPF_ERRNO__INTERNAL; 1233 } 1234 1235 prog->instances.fds = malloc(sizeof(int)); 1236 if (!prog->instances.fds) { 1237 pr_warning("Not enough memory for BPF fds\n"); 1238 return -ENOMEM; 1239 } 1240 prog->instances.nr = 1; 1241 prog->instances.fds[0] = -1; 1242 } 1243 1244 if (!prog->preprocessor) { 1245 if (prog->instances.nr != 1) { 1246 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 1247 prog->section_name, prog->instances.nr); 1248 } 1249 err = load_program(prog->type, prog->name, prog->insns, 1250 prog->insns_cnt, license, kern_version, &fd); 1251 if (!err) 1252 prog->instances.fds[0] = fd; 1253 goto out; 1254 } 1255 1256 for (i = 0; i < prog->instances.nr; i++) { 1257 struct bpf_prog_prep_result result; 1258 bpf_program_prep_t preprocessor = prog->preprocessor; 1259 1260 bzero(&result, sizeof(result)); 1261 err = preprocessor(prog, i, prog->insns, 1262 prog->insns_cnt, &result); 1263 if (err) { 1264 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 1265 i, prog->section_name); 1266 goto out; 1267 } 1268 1269 if (!result.new_insn_ptr || !result.new_insn_cnt) { 1270 pr_debug("Skip loading the %dth instance of program '%s'\n", 1271 i, prog->section_name); 1272 prog->instances.fds[i] = -1; 1273 if (result.pfd) 1274 *result.pfd = -1; 1275 continue; 1276 } 1277 1278 err = load_program(prog->type, prog->name, 1279 result.new_insn_ptr, 1280 result.new_insn_cnt, 1281 license, kern_version, &fd); 1282 1283 if (err) { 1284 pr_warning("Loading the %dth instance of program '%s' failed\n", 1285 i, prog->section_name); 1286 goto out; 1287 } 1288 1289 if (result.pfd) 1290 *result.pfd = fd; 1291 prog->instances.fds[i] = fd; 1292 } 1293 out: 1294 if (err) 1295 pr_warning("failed to load program '%s'\n", 1296 prog->section_name); 1297 zfree(&prog->insns); 1298 prog->insns_cnt = 0; 1299 return err; 1300 } 1301 1302 static int 1303 bpf_object__load_progs(struct bpf_object *obj) 1304 { 1305 size_t i; 1306 int err; 1307 1308 for (i = 0; i < obj->nr_programs; i++) { 1309 if (obj->programs[i].idx == obj->efile.text_shndx) 1310 continue; 1311 err = bpf_program__load(&obj->programs[i], 1312 obj->license, 1313 obj->kern_version); 1314 if (err) 1315 return err; 1316 } 1317 return 0; 1318 } 1319 1320 static int bpf_object__validate(struct bpf_object *obj) 1321 { 1322 if (obj->kern_version == 0) { 1323 pr_warning("%s doesn't provide kernel version\n", 1324 obj->path); 1325 return -LIBBPF_ERRNO__KVERSION; 1326 } 1327 return 0; 1328 } 1329 1330 static struct bpf_object * 1331 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) 1332 { 1333 struct bpf_object *obj; 1334 int err; 1335 1336 if (elf_version(EV_CURRENT) == EV_NONE) { 1337 pr_warning("failed to init libelf for %s\n", path); 1338 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 1339 } 1340 1341 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 1342 if (IS_ERR(obj)) 1343 return obj; 1344 1345 CHECK_ERR(bpf_object__elf_init(obj), err, out); 1346 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 1347 CHECK_ERR(bpf_object__elf_collect(obj), err, out); 1348 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 1349 CHECK_ERR(bpf_object__validate(obj), err, out); 1350 1351 bpf_object__elf_finish(obj); 1352 return obj; 1353 out: 1354 bpf_object__close(obj); 1355 return ERR_PTR(err); 1356 } 1357 1358 struct bpf_object *bpf_object__open(const char *path) 1359 { 1360 /* param validation */ 1361 if (!path) 1362 return NULL; 1363 1364 pr_debug("loading %s\n", path); 1365 1366 return __bpf_object__open(path, NULL, 0); 1367 } 1368 1369 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 1370 size_t obj_buf_sz, 1371 const char *name) 1372 { 1373 char tmp_name[64]; 1374 1375 /* param validation */ 1376 if (!obj_buf || obj_buf_sz <= 0) 1377 return NULL; 1378 1379 if (!name) { 1380 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 1381 (unsigned long)obj_buf, 1382 (unsigned long)obj_buf_sz); 1383 tmp_name[sizeof(tmp_name) - 1] = '\0'; 1384 name = tmp_name; 1385 } 1386 pr_debug("loading object '%s' from buffer\n", 1387 name); 1388 1389 return __bpf_object__open(name, obj_buf, obj_buf_sz); 1390 } 1391 1392 int bpf_object__unload(struct bpf_object *obj) 1393 { 1394 size_t i; 1395 1396 if (!obj) 1397 return -EINVAL; 1398 1399 for (i = 0; i < obj->nr_maps; i++) 1400 zclose(obj->maps[i].fd); 1401 1402 for (i = 0; i < obj->nr_programs; i++) 1403 bpf_program__unload(&obj->programs[i]); 1404 1405 return 0; 1406 } 1407 1408 int bpf_object__load(struct bpf_object *obj) 1409 { 1410 int err; 1411 1412 if (!obj) 1413 return -EINVAL; 1414 1415 if (obj->loaded) { 1416 pr_warning("object should not be loaded twice\n"); 1417 return -EINVAL; 1418 } 1419 1420 obj->loaded = true; 1421 1422 CHECK_ERR(bpf_object__create_maps(obj), err, out); 1423 CHECK_ERR(bpf_object__relocate(obj), err, out); 1424 CHECK_ERR(bpf_object__load_progs(obj), err, out); 1425 1426 return 0; 1427 out: 1428 bpf_object__unload(obj); 1429 pr_warning("failed to load object '%s'\n", obj->path); 1430 return err; 1431 } 1432 1433 static int check_path(const char *path) 1434 { 1435 struct statfs st_fs; 1436 char *dname, *dir; 1437 int err = 0; 1438 1439 if (path == NULL) 1440 return -EINVAL; 1441 1442 dname = strdup(path); 1443 if (dname == NULL) 1444 return -ENOMEM; 1445 1446 dir = dirname(dname); 1447 if (statfs(dir, &st_fs)) { 1448 pr_warning("failed to statfs %s: %s\n", dir, strerror(errno)); 1449 err = -errno; 1450 } 1451 free(dname); 1452 1453 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 1454 pr_warning("specified path %s is not on BPF FS\n", path); 1455 err = -EINVAL; 1456 } 1457 1458 return err; 1459 } 1460 1461 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 1462 int instance) 1463 { 1464 int err; 1465 1466 err = check_path(path); 1467 if (err) 1468 return err; 1469 1470 if (prog == NULL) { 1471 pr_warning("invalid program pointer\n"); 1472 return -EINVAL; 1473 } 1474 1475 if (instance < 0 || instance >= prog->instances.nr) { 1476 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 1477 instance, prog->section_name, prog->instances.nr); 1478 return -EINVAL; 1479 } 1480 1481 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 1482 pr_warning("failed to pin program: %s\n", strerror(errno)); 1483 return -errno; 1484 } 1485 pr_debug("pinned program '%s'\n", path); 1486 1487 return 0; 1488 } 1489 1490 static int make_dir(const char *path) 1491 { 1492 int err = 0; 1493 1494 if (mkdir(path, 0700) && errno != EEXIST) 1495 err = -errno; 1496 1497 if (err) 1498 pr_warning("failed to mkdir %s: %s\n", path, strerror(-err)); 1499 return err; 1500 } 1501 1502 int bpf_program__pin(struct bpf_program *prog, const char *path) 1503 { 1504 int i, err; 1505 1506 err = check_path(path); 1507 if (err) 1508 return err; 1509 1510 if (prog == NULL) { 1511 pr_warning("invalid program pointer\n"); 1512 return -EINVAL; 1513 } 1514 1515 if (prog->instances.nr <= 0) { 1516 pr_warning("no instances of prog %s to pin\n", 1517 prog->section_name); 1518 return -EINVAL; 1519 } 1520 1521 err = make_dir(path); 1522 if (err) 1523 return err; 1524 1525 for (i = 0; i < prog->instances.nr; i++) { 1526 char buf[PATH_MAX]; 1527 int len; 1528 1529 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 1530 if (len < 0) 1531 return -EINVAL; 1532 else if (len >= PATH_MAX) 1533 return -ENAMETOOLONG; 1534 1535 err = bpf_program__pin_instance(prog, buf, i); 1536 if (err) 1537 return err; 1538 } 1539 1540 return 0; 1541 } 1542 1543 int bpf_map__pin(struct bpf_map *map, const char *path) 1544 { 1545 int err; 1546 1547 err = check_path(path); 1548 if (err) 1549 return err; 1550 1551 if (map == NULL) { 1552 pr_warning("invalid map pointer\n"); 1553 return -EINVAL; 1554 } 1555 1556 if (bpf_obj_pin(map->fd, path)) { 1557 pr_warning("failed to pin map: %s\n", strerror(errno)); 1558 return -errno; 1559 } 1560 1561 pr_debug("pinned map '%s'\n", path); 1562 return 0; 1563 } 1564 1565 int bpf_object__pin(struct bpf_object *obj, const char *path) 1566 { 1567 struct bpf_program *prog; 1568 struct bpf_map *map; 1569 int err; 1570 1571 if (!obj) 1572 return -ENOENT; 1573 1574 if (!obj->loaded) { 1575 pr_warning("object not yet loaded; load it first\n"); 1576 return -ENOENT; 1577 } 1578 1579 err = make_dir(path); 1580 if (err) 1581 return err; 1582 1583 bpf_map__for_each(map, obj) { 1584 char buf[PATH_MAX]; 1585 int len; 1586 1587 len = snprintf(buf, PATH_MAX, "%s/%s", path, 1588 bpf_map__name(map)); 1589 if (len < 0) 1590 return -EINVAL; 1591 else if (len >= PATH_MAX) 1592 return -ENAMETOOLONG; 1593 1594 err = bpf_map__pin(map, buf); 1595 if (err) 1596 return err; 1597 } 1598 1599 bpf_object__for_each_program(prog, obj) { 1600 char buf[PATH_MAX]; 1601 int len; 1602 1603 len = snprintf(buf, PATH_MAX, "%s/%s", path, 1604 prog->section_name); 1605 if (len < 0) 1606 return -EINVAL; 1607 else if (len >= PATH_MAX) 1608 return -ENAMETOOLONG; 1609 1610 err = bpf_program__pin(prog, buf); 1611 if (err) 1612 return err; 1613 } 1614 1615 return 0; 1616 } 1617 1618 void bpf_object__close(struct bpf_object *obj) 1619 { 1620 size_t i; 1621 1622 if (!obj) 1623 return; 1624 1625 if (obj->clear_priv) 1626 obj->clear_priv(obj, obj->priv); 1627 1628 bpf_object__elf_finish(obj); 1629 bpf_object__unload(obj); 1630 1631 for (i = 0; i < obj->nr_maps; i++) { 1632 zfree(&obj->maps[i].name); 1633 if (obj->maps[i].clear_priv) 1634 obj->maps[i].clear_priv(&obj->maps[i], 1635 obj->maps[i].priv); 1636 obj->maps[i].priv = NULL; 1637 obj->maps[i].clear_priv = NULL; 1638 } 1639 zfree(&obj->maps); 1640 obj->nr_maps = 0; 1641 1642 if (obj->programs && obj->nr_programs) { 1643 for (i = 0; i < obj->nr_programs; i++) 1644 bpf_program__exit(&obj->programs[i]); 1645 } 1646 zfree(&obj->programs); 1647 1648 list_del(&obj->list); 1649 free(obj); 1650 } 1651 1652 struct bpf_object * 1653 bpf_object__next(struct bpf_object *prev) 1654 { 1655 struct bpf_object *next; 1656 1657 if (!prev) 1658 next = list_first_entry(&bpf_objects_list, 1659 struct bpf_object, 1660 list); 1661 else 1662 next = list_next_entry(prev, list); 1663 1664 /* Empty list is noticed here so don't need checking on entry. */ 1665 if (&next->list == &bpf_objects_list) 1666 return NULL; 1667 1668 return next; 1669 } 1670 1671 const char *bpf_object__name(struct bpf_object *obj) 1672 { 1673 return obj ? obj->path : ERR_PTR(-EINVAL); 1674 } 1675 1676 unsigned int bpf_object__kversion(struct bpf_object *obj) 1677 { 1678 return obj ? obj->kern_version : 0; 1679 } 1680 1681 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 1682 bpf_object_clear_priv_t clear_priv) 1683 { 1684 if (obj->priv && obj->clear_priv) 1685 obj->clear_priv(obj, obj->priv); 1686 1687 obj->priv = priv; 1688 obj->clear_priv = clear_priv; 1689 return 0; 1690 } 1691 1692 void *bpf_object__priv(struct bpf_object *obj) 1693 { 1694 return obj ? obj->priv : ERR_PTR(-EINVAL); 1695 } 1696 1697 struct bpf_program * 1698 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 1699 { 1700 size_t idx; 1701 1702 if (!obj->programs) 1703 return NULL; 1704 /* First handler */ 1705 if (prev == NULL) 1706 return &obj->programs[0]; 1707 1708 if (prev->obj != obj) { 1709 pr_warning("error: program handler doesn't match object\n"); 1710 return NULL; 1711 } 1712 1713 idx = (prev - obj->programs) + 1; 1714 if (idx >= obj->nr_programs) 1715 return NULL; 1716 return &obj->programs[idx]; 1717 } 1718 1719 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 1720 bpf_program_clear_priv_t clear_priv) 1721 { 1722 if (prog->priv && prog->clear_priv) 1723 prog->clear_priv(prog, prog->priv); 1724 1725 prog->priv = priv; 1726 prog->clear_priv = clear_priv; 1727 return 0; 1728 } 1729 1730 void *bpf_program__priv(struct bpf_program *prog) 1731 { 1732 return prog ? prog->priv : ERR_PTR(-EINVAL); 1733 } 1734 1735 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 1736 { 1737 const char *title; 1738 1739 title = prog->section_name; 1740 if (needs_copy) { 1741 title = strdup(title); 1742 if (!title) { 1743 pr_warning("failed to strdup program title\n"); 1744 return ERR_PTR(-ENOMEM); 1745 } 1746 } 1747 1748 return title; 1749 } 1750 1751 int bpf_program__fd(struct bpf_program *prog) 1752 { 1753 return bpf_program__nth_fd(prog, 0); 1754 } 1755 1756 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 1757 bpf_program_prep_t prep) 1758 { 1759 int *instances_fds; 1760 1761 if (nr_instances <= 0 || !prep) 1762 return -EINVAL; 1763 1764 if (prog->instances.nr > 0 || prog->instances.fds) { 1765 pr_warning("Can't set pre-processor after loading\n"); 1766 return -EINVAL; 1767 } 1768 1769 instances_fds = malloc(sizeof(int) * nr_instances); 1770 if (!instances_fds) { 1771 pr_warning("alloc memory failed for fds\n"); 1772 return -ENOMEM; 1773 } 1774 1775 /* fill all fd with -1 */ 1776 memset(instances_fds, -1, sizeof(int) * nr_instances); 1777 1778 prog->instances.nr = nr_instances; 1779 prog->instances.fds = instances_fds; 1780 prog->preprocessor = prep; 1781 return 0; 1782 } 1783 1784 int bpf_program__nth_fd(struct bpf_program *prog, int n) 1785 { 1786 int fd; 1787 1788 if (n >= prog->instances.nr || n < 0) { 1789 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 1790 n, prog->section_name, prog->instances.nr); 1791 return -EINVAL; 1792 } 1793 1794 fd = prog->instances.fds[n]; 1795 if (fd < 0) { 1796 pr_warning("%dth instance of program '%s' is invalid\n", 1797 n, prog->section_name); 1798 return -ENOENT; 1799 } 1800 1801 return fd; 1802 } 1803 1804 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 1805 { 1806 prog->type = type; 1807 } 1808 1809 static bool bpf_program__is_type(struct bpf_program *prog, 1810 enum bpf_prog_type type) 1811 { 1812 return prog ? (prog->type == type) : false; 1813 } 1814 1815 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 1816 int bpf_program__set_##NAME(struct bpf_program *prog) \ 1817 { \ 1818 if (!prog) \ 1819 return -EINVAL; \ 1820 bpf_program__set_type(prog, TYPE); \ 1821 return 0; \ 1822 } \ 1823 \ 1824 bool bpf_program__is_##NAME(struct bpf_program *prog) \ 1825 { \ 1826 return bpf_program__is_type(prog, TYPE); \ 1827 } \ 1828 1829 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 1830 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 1831 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 1832 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 1833 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 1834 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 1835 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 1836 1837 #define BPF_PROG_SEC(string, type) { string, sizeof(string) - 1, type } 1838 static const struct { 1839 const char *sec; 1840 size_t len; 1841 enum bpf_prog_type prog_type; 1842 } section_names[] = { 1843 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 1844 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 1845 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 1846 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 1847 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 1848 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 1849 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 1850 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 1851 BPF_PROG_SEC("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 1852 BPF_PROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK), 1853 BPF_PROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE), 1854 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 1855 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 1856 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 1857 BPF_PROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS), 1858 BPF_PROG_SEC("sk_skb", BPF_PROG_TYPE_SK_SKB), 1859 }; 1860 #undef BPF_PROG_SEC 1861 1862 static enum bpf_prog_type bpf_program__guess_type(struct bpf_program *prog) 1863 { 1864 int i; 1865 1866 if (!prog->section_name) 1867 goto err; 1868 1869 for (i = 0; i < ARRAY_SIZE(section_names); i++) 1870 if (strncmp(prog->section_name, section_names[i].sec, 1871 section_names[i].len) == 0) 1872 return section_names[i].prog_type; 1873 1874 err: 1875 pr_warning("failed to guess program type based on section name %s\n", 1876 prog->section_name); 1877 1878 return BPF_PROG_TYPE_UNSPEC; 1879 } 1880 1881 int bpf_map__fd(struct bpf_map *map) 1882 { 1883 return map ? map->fd : -EINVAL; 1884 } 1885 1886 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 1887 { 1888 return map ? &map->def : ERR_PTR(-EINVAL); 1889 } 1890 1891 const char *bpf_map__name(struct bpf_map *map) 1892 { 1893 return map ? map->name : NULL; 1894 } 1895 1896 int bpf_map__set_priv(struct bpf_map *map, void *priv, 1897 bpf_map_clear_priv_t clear_priv) 1898 { 1899 if (!map) 1900 return -EINVAL; 1901 1902 if (map->priv) { 1903 if (map->clear_priv) 1904 map->clear_priv(map, map->priv); 1905 } 1906 1907 map->priv = priv; 1908 map->clear_priv = clear_priv; 1909 return 0; 1910 } 1911 1912 void *bpf_map__priv(struct bpf_map *map) 1913 { 1914 return map ? map->priv : ERR_PTR(-EINVAL); 1915 } 1916 1917 struct bpf_map * 1918 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 1919 { 1920 size_t idx; 1921 struct bpf_map *s, *e; 1922 1923 if (!obj || !obj->maps) 1924 return NULL; 1925 1926 s = obj->maps; 1927 e = obj->maps + obj->nr_maps; 1928 1929 if (prev == NULL) 1930 return s; 1931 1932 if ((prev < s) || (prev >= e)) { 1933 pr_warning("error in %s: map handler doesn't belong to object\n", 1934 __func__); 1935 return NULL; 1936 } 1937 1938 idx = (prev - obj->maps) + 1; 1939 if (idx >= obj->nr_maps) 1940 return NULL; 1941 return &obj->maps[idx]; 1942 } 1943 1944 struct bpf_map * 1945 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 1946 { 1947 struct bpf_map *pos; 1948 1949 bpf_map__for_each(pos, obj) { 1950 if (pos->name && !strcmp(pos->name, name)) 1951 return pos; 1952 } 1953 return NULL; 1954 } 1955 1956 struct bpf_map * 1957 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 1958 { 1959 int i; 1960 1961 for (i = 0; i < obj->nr_maps; i++) { 1962 if (obj->maps[i].offset == offset) 1963 return &obj->maps[i]; 1964 } 1965 return ERR_PTR(-ENOENT); 1966 } 1967 1968 long libbpf_get_error(const void *ptr) 1969 { 1970 if (IS_ERR(ptr)) 1971 return PTR_ERR(ptr); 1972 return 0; 1973 } 1974 1975 int bpf_prog_load(const char *file, enum bpf_prog_type type, 1976 struct bpf_object **pobj, int *prog_fd) 1977 { 1978 struct bpf_program *prog, *first_prog = NULL; 1979 struct bpf_object *obj; 1980 int err; 1981 1982 obj = bpf_object__open(file); 1983 if (IS_ERR(obj)) 1984 return -ENOENT; 1985 1986 bpf_object__for_each_program(prog, obj) { 1987 /* 1988 * If type is not specified, try to guess it based on 1989 * section name. 1990 */ 1991 if (type == BPF_PROG_TYPE_UNSPEC) { 1992 type = bpf_program__guess_type(prog); 1993 if (type == BPF_PROG_TYPE_UNSPEC) { 1994 bpf_object__close(obj); 1995 return -EINVAL; 1996 } 1997 } 1998 1999 bpf_program__set_type(prog, type); 2000 if (prog->idx != obj->efile.text_shndx && !first_prog) 2001 first_prog = prog; 2002 } 2003 2004 if (!first_prog) { 2005 pr_warning("object file doesn't contain bpf program\n"); 2006 bpf_object__close(obj); 2007 return -ENOENT; 2008 } 2009 2010 err = bpf_object__load(obj); 2011 if (err) { 2012 bpf_object__close(obj); 2013 return -EINVAL; 2014 } 2015 2016 *pobj = obj; 2017 *prog_fd = bpf_program__fd(first_prog); 2018 return 0; 2019 } 2020