1 // SPDX-License-Identifier: LGPL-2.1 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * 11 * This program is free software; you can redistribute it and/or 12 * modify it under the terms of the GNU Lesser General Public 13 * License as published by the Free Software Foundation; 14 * version 2.1 of the License (not later!) 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU Lesser General Public License for more details. 20 * 21 * You should have received a copy of the GNU Lesser General Public 22 * License along with this program; if not, see <http://www.gnu.org/licenses> 23 */ 24 25 #include <stdlib.h> 26 #include <stdio.h> 27 #include <stdarg.h> 28 #include <libgen.h> 29 #include <inttypes.h> 30 #include <string.h> 31 #include <unistd.h> 32 #include <fcntl.h> 33 #include <errno.h> 34 #include <asm/unistd.h> 35 #include <linux/err.h> 36 #include <linux/kernel.h> 37 #include <linux/bpf.h> 38 #include <linux/list.h> 39 #include <linux/limits.h> 40 #include <sys/stat.h> 41 #include <sys/types.h> 42 #include <sys/vfs.h> 43 #include <libelf.h> 44 #include <gelf.h> 45 46 #include "libbpf.h" 47 #include "bpf.h" 48 49 #ifndef EM_BPF 50 #define EM_BPF 247 51 #endif 52 53 #ifndef BPF_FS_MAGIC 54 #define BPF_FS_MAGIC 0xcafe4a11 55 #endif 56 57 #define __printf(a, b) __attribute__((format(printf, a, b))) 58 59 __printf(1, 2) 60 static int __base_pr(const char *format, ...) 61 { 62 va_list args; 63 int err; 64 65 va_start(args, format); 66 err = vfprintf(stderr, format, args); 67 va_end(args); 68 return err; 69 } 70 71 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; 72 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; 73 static __printf(1, 2) libbpf_print_fn_t __pr_debug; 74 75 #define __pr(func, fmt, ...) \ 76 do { \ 77 if ((func)) \ 78 (func)("libbpf: " fmt, ##__VA_ARGS__); \ 79 } while (0) 80 81 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) 82 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) 83 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) 84 85 void libbpf_set_print(libbpf_print_fn_t warn, 86 libbpf_print_fn_t info, 87 libbpf_print_fn_t debug) 88 { 89 __pr_warning = warn; 90 __pr_info = info; 91 __pr_debug = debug; 92 } 93 94 #define STRERR_BUFSIZE 128 95 96 #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) 97 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) 98 #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) 99 100 static const char *libbpf_strerror_table[NR_ERRNO] = { 101 [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", 102 [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", 103 [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", 104 [ERRCODE_OFFSET(ENDIAN)] = "Endian mismatch", 105 [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", 106 [ERRCODE_OFFSET(RELOC)] = "Relocation failed", 107 [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", 108 [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", 109 [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", 110 [ERRCODE_OFFSET(PROGTYPE)] = "Kernel doesn't support this program type", 111 [ERRCODE_OFFSET(WRNGPID)] = "Wrong pid in netlink message", 112 [ERRCODE_OFFSET(INVSEQ)] = "Invalid netlink sequence", 113 }; 114 115 int libbpf_strerror(int err, char *buf, size_t size) 116 { 117 if (!buf || !size) 118 return -1; 119 120 err = err > 0 ? err : -err; 121 122 if (err < __LIBBPF_ERRNO__START) { 123 int ret; 124 125 ret = strerror_r(err, buf, size); 126 buf[size - 1] = '\0'; 127 return ret; 128 } 129 130 if (err < __LIBBPF_ERRNO__END) { 131 const char *msg; 132 133 msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; 134 snprintf(buf, size, "%s", msg); 135 buf[size - 1] = '\0'; 136 return 0; 137 } 138 139 snprintf(buf, size, "Unknown libbpf error %d", err); 140 buf[size - 1] = '\0'; 141 return -1; 142 } 143 144 #define CHECK_ERR(action, err, out) do { \ 145 err = action; \ 146 if (err) \ 147 goto out; \ 148 } while(0) 149 150 151 /* Copied from tools/perf/util/util.h */ 152 #ifndef zfree 153 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 154 #endif 155 156 #ifndef zclose 157 # define zclose(fd) ({ \ 158 int ___err = 0; \ 159 if ((fd) >= 0) \ 160 ___err = close((fd)); \ 161 fd = -1; \ 162 ___err; }) 163 #endif 164 165 #ifdef HAVE_LIBELF_MMAP_SUPPORT 166 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 167 #else 168 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 169 #endif 170 171 /* 172 * bpf_prog should be a better name but it has been used in 173 * linux/filter.h. 174 */ 175 struct bpf_program { 176 /* Index in elf obj file, for relocation use. */ 177 int idx; 178 char *name; 179 char *section_name; 180 struct bpf_insn *insns; 181 size_t insns_cnt, main_prog_cnt; 182 enum bpf_prog_type type; 183 184 struct reloc_desc { 185 enum { 186 RELO_LD64, 187 RELO_CALL, 188 } type; 189 int insn_idx; 190 union { 191 int map_idx; 192 int text_off; 193 }; 194 } *reloc_desc; 195 int nr_reloc; 196 197 struct { 198 int nr; 199 int *fds; 200 } instances; 201 bpf_program_prep_t preprocessor; 202 203 struct bpf_object *obj; 204 void *priv; 205 bpf_program_clear_priv_t clear_priv; 206 }; 207 208 struct bpf_map { 209 int fd; 210 char *name; 211 size_t offset; 212 struct bpf_map_def def; 213 void *priv; 214 bpf_map_clear_priv_t clear_priv; 215 }; 216 217 static LIST_HEAD(bpf_objects_list); 218 219 struct bpf_object { 220 char license[64]; 221 u32 kern_version; 222 223 struct bpf_program *programs; 224 size_t nr_programs; 225 struct bpf_map *maps; 226 size_t nr_maps; 227 228 bool loaded; 229 230 /* 231 * Information when doing elf related work. Only valid if fd 232 * is valid. 233 */ 234 struct { 235 int fd; 236 void *obj_buf; 237 size_t obj_buf_sz; 238 Elf *elf; 239 GElf_Ehdr ehdr; 240 Elf_Data *symbols; 241 size_t strtabidx; 242 struct { 243 GElf_Shdr shdr; 244 Elf_Data *data; 245 } *reloc; 246 int nr_reloc; 247 int maps_shndx; 248 int text_shndx; 249 } efile; 250 /* 251 * All loaded bpf_object is linked in a list, which is 252 * hidden to caller. bpf_objects__<func> handlers deal with 253 * all objects. 254 */ 255 struct list_head list; 256 257 void *priv; 258 bpf_object_clear_priv_t clear_priv; 259 260 char path[]; 261 }; 262 #define obj_elf_valid(o) ((o)->efile.elf) 263 264 static void bpf_program__unload(struct bpf_program *prog) 265 { 266 int i; 267 268 if (!prog) 269 return; 270 271 /* 272 * If the object is opened but the program was never loaded, 273 * it is possible that prog->instances.nr == -1. 274 */ 275 if (prog->instances.nr > 0) { 276 for (i = 0; i < prog->instances.nr; i++) 277 zclose(prog->instances.fds[i]); 278 } else if (prog->instances.nr != -1) { 279 pr_warning("Internal error: instances.nr is %d\n", 280 prog->instances.nr); 281 } 282 283 prog->instances.nr = -1; 284 zfree(&prog->instances.fds); 285 } 286 287 static void bpf_program__exit(struct bpf_program *prog) 288 { 289 if (!prog) 290 return; 291 292 if (prog->clear_priv) 293 prog->clear_priv(prog, prog->priv); 294 295 prog->priv = NULL; 296 prog->clear_priv = NULL; 297 298 bpf_program__unload(prog); 299 zfree(&prog->name); 300 zfree(&prog->section_name); 301 zfree(&prog->insns); 302 zfree(&prog->reloc_desc); 303 304 prog->nr_reloc = 0; 305 prog->insns_cnt = 0; 306 prog->idx = -1; 307 } 308 309 static int 310 bpf_program__init(void *data, size_t size, char *section_name, int idx, 311 struct bpf_program *prog) 312 { 313 if (size < sizeof(struct bpf_insn)) { 314 pr_warning("corrupted section '%s'\n", section_name); 315 return -EINVAL; 316 } 317 318 bzero(prog, sizeof(*prog)); 319 320 prog->section_name = strdup(section_name); 321 if (!prog->section_name) { 322 pr_warning("failed to alloc name for prog under section %s\n", 323 section_name); 324 goto errout; 325 } 326 327 prog->insns = malloc(size); 328 if (!prog->insns) { 329 pr_warning("failed to alloc insns for prog under section %s\n", 330 section_name); 331 goto errout; 332 } 333 prog->insns_cnt = size / sizeof(struct bpf_insn); 334 memcpy(prog->insns, data, 335 prog->insns_cnt * sizeof(struct bpf_insn)); 336 prog->idx = idx; 337 prog->instances.fds = NULL; 338 prog->instances.nr = -1; 339 prog->type = BPF_PROG_TYPE_KPROBE; 340 341 return 0; 342 errout: 343 bpf_program__exit(prog); 344 return -ENOMEM; 345 } 346 347 static int 348 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 349 char *section_name, int idx) 350 { 351 struct bpf_program prog, *progs; 352 int nr_progs, err; 353 354 err = bpf_program__init(data, size, section_name, idx, &prog); 355 if (err) 356 return err; 357 358 progs = obj->programs; 359 nr_progs = obj->nr_programs; 360 361 progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); 362 if (!progs) { 363 /* 364 * In this case the original obj->programs 365 * is still valid, so don't need special treat for 366 * bpf_close_object(). 367 */ 368 pr_warning("failed to alloc a new program under section '%s'\n", 369 section_name); 370 bpf_program__exit(&prog); 371 return -ENOMEM; 372 } 373 374 pr_debug("found program %s\n", prog.section_name); 375 obj->programs = progs; 376 obj->nr_programs = nr_progs + 1; 377 prog.obj = obj; 378 progs[nr_progs] = prog; 379 return 0; 380 } 381 382 static int 383 bpf_object__init_prog_names(struct bpf_object *obj) 384 { 385 Elf_Data *symbols = obj->efile.symbols; 386 struct bpf_program *prog; 387 size_t pi, si; 388 389 for (pi = 0; pi < obj->nr_programs; pi++) { 390 const char *name = NULL; 391 392 prog = &obj->programs[pi]; 393 if (prog->idx == obj->efile.text_shndx) { 394 name = ".text"; 395 goto skip_search; 396 } 397 398 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 399 si++) { 400 GElf_Sym sym; 401 402 if (!gelf_getsym(symbols, si, &sym)) 403 continue; 404 if (sym.st_shndx != prog->idx) 405 continue; 406 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) 407 continue; 408 409 name = elf_strptr(obj->efile.elf, 410 obj->efile.strtabidx, 411 sym.st_name); 412 if (!name) { 413 pr_warning("failed to get sym name string for prog %s\n", 414 prog->section_name); 415 return -LIBBPF_ERRNO__LIBELF; 416 } 417 } 418 419 if (!name) { 420 pr_warning("failed to find sym for prog %s\n", 421 prog->section_name); 422 return -EINVAL; 423 } 424 skip_search: 425 prog->name = strdup(name); 426 if (!prog->name) { 427 pr_warning("failed to allocate memory for prog sym %s\n", 428 name); 429 return -ENOMEM; 430 } 431 } 432 433 return 0; 434 } 435 436 static struct bpf_object *bpf_object__new(const char *path, 437 void *obj_buf, 438 size_t obj_buf_sz) 439 { 440 struct bpf_object *obj; 441 442 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 443 if (!obj) { 444 pr_warning("alloc memory failed for %s\n", path); 445 return ERR_PTR(-ENOMEM); 446 } 447 448 strcpy(obj->path, path); 449 obj->efile.fd = -1; 450 451 /* 452 * Caller of this function should also calls 453 * bpf_object__elf_finish() after data collection to return 454 * obj_buf to user. If not, we should duplicate the buffer to 455 * avoid user freeing them before elf finish. 456 */ 457 obj->efile.obj_buf = obj_buf; 458 obj->efile.obj_buf_sz = obj_buf_sz; 459 obj->efile.maps_shndx = -1; 460 461 obj->loaded = false; 462 463 INIT_LIST_HEAD(&obj->list); 464 list_add(&obj->list, &bpf_objects_list); 465 return obj; 466 } 467 468 static void bpf_object__elf_finish(struct bpf_object *obj) 469 { 470 if (!obj_elf_valid(obj)) 471 return; 472 473 if (obj->efile.elf) { 474 elf_end(obj->efile.elf); 475 obj->efile.elf = NULL; 476 } 477 obj->efile.symbols = NULL; 478 479 zfree(&obj->efile.reloc); 480 obj->efile.nr_reloc = 0; 481 zclose(obj->efile.fd); 482 obj->efile.obj_buf = NULL; 483 obj->efile.obj_buf_sz = 0; 484 } 485 486 static int bpf_object__elf_init(struct bpf_object *obj) 487 { 488 int err = 0; 489 GElf_Ehdr *ep; 490 491 if (obj_elf_valid(obj)) { 492 pr_warning("elf init: internal error\n"); 493 return -LIBBPF_ERRNO__LIBELF; 494 } 495 496 if (obj->efile.obj_buf_sz > 0) { 497 /* 498 * obj_buf should have been validated by 499 * bpf_object__open_buffer(). 500 */ 501 obj->efile.elf = elf_memory(obj->efile.obj_buf, 502 obj->efile.obj_buf_sz); 503 } else { 504 obj->efile.fd = open(obj->path, O_RDONLY); 505 if (obj->efile.fd < 0) { 506 pr_warning("failed to open %s: %s\n", obj->path, 507 strerror(errno)); 508 return -errno; 509 } 510 511 obj->efile.elf = elf_begin(obj->efile.fd, 512 LIBBPF_ELF_C_READ_MMAP, 513 NULL); 514 } 515 516 if (!obj->efile.elf) { 517 pr_warning("failed to open %s as ELF file\n", 518 obj->path); 519 err = -LIBBPF_ERRNO__LIBELF; 520 goto errout; 521 } 522 523 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 524 pr_warning("failed to get EHDR from %s\n", 525 obj->path); 526 err = -LIBBPF_ERRNO__FORMAT; 527 goto errout; 528 } 529 ep = &obj->efile.ehdr; 530 531 /* Old LLVM set e_machine to EM_NONE */ 532 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { 533 pr_warning("%s is not an eBPF object file\n", 534 obj->path); 535 err = -LIBBPF_ERRNO__FORMAT; 536 goto errout; 537 } 538 539 return 0; 540 errout: 541 bpf_object__elf_finish(obj); 542 return err; 543 } 544 545 static int 546 bpf_object__check_endianness(struct bpf_object *obj) 547 { 548 static unsigned int const endian = 1; 549 550 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 551 case ELFDATA2LSB: 552 /* We are big endian, BPF obj is little endian. */ 553 if (*(unsigned char const *)&endian != 1) 554 goto mismatch; 555 break; 556 557 case ELFDATA2MSB: 558 /* We are little endian, BPF obj is big endian. */ 559 if (*(unsigned char const *)&endian != 0) 560 goto mismatch; 561 break; 562 default: 563 return -LIBBPF_ERRNO__ENDIAN; 564 } 565 566 return 0; 567 568 mismatch: 569 pr_warning("Error: endianness mismatch.\n"); 570 return -LIBBPF_ERRNO__ENDIAN; 571 } 572 573 static int 574 bpf_object__init_license(struct bpf_object *obj, 575 void *data, size_t size) 576 { 577 memcpy(obj->license, data, 578 min(size, sizeof(obj->license) - 1)); 579 pr_debug("license of %s is %s\n", obj->path, obj->license); 580 return 0; 581 } 582 583 static int 584 bpf_object__init_kversion(struct bpf_object *obj, 585 void *data, size_t size) 586 { 587 u32 kver; 588 589 if (size != sizeof(kver)) { 590 pr_warning("invalid kver section in %s\n", obj->path); 591 return -LIBBPF_ERRNO__FORMAT; 592 } 593 memcpy(&kver, data, sizeof(kver)); 594 obj->kern_version = kver; 595 pr_debug("kernel version of %s is %x\n", obj->path, 596 obj->kern_version); 597 return 0; 598 } 599 600 static int compare_bpf_map(const void *_a, const void *_b) 601 { 602 const struct bpf_map *a = _a; 603 const struct bpf_map *b = _b; 604 605 return a->offset - b->offset; 606 } 607 608 static int 609 bpf_object__init_maps(struct bpf_object *obj) 610 { 611 int i, map_idx, map_def_sz, nr_maps = 0; 612 Elf_Scn *scn; 613 Elf_Data *data; 614 Elf_Data *symbols = obj->efile.symbols; 615 616 if (obj->efile.maps_shndx < 0) 617 return -EINVAL; 618 if (!symbols) 619 return -EINVAL; 620 621 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); 622 if (scn) 623 data = elf_getdata(scn, NULL); 624 if (!scn || !data) { 625 pr_warning("failed to get Elf_Data from map section %d\n", 626 obj->efile.maps_shndx); 627 return -EINVAL; 628 } 629 630 /* 631 * Count number of maps. Each map has a name. 632 * Array of maps is not supported: only the first element is 633 * considered. 634 * 635 * TODO: Detect array of map and report error. 636 */ 637 for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 638 GElf_Sym sym; 639 640 if (!gelf_getsym(symbols, i, &sym)) 641 continue; 642 if (sym.st_shndx != obj->efile.maps_shndx) 643 continue; 644 nr_maps++; 645 } 646 647 /* Alloc obj->maps and fill nr_maps. */ 648 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 649 nr_maps, data->d_size); 650 651 if (!nr_maps) 652 return 0; 653 654 /* Assume equally sized map definitions */ 655 map_def_sz = data->d_size / nr_maps; 656 if (!data->d_size || (data->d_size % nr_maps) != 0) { 657 pr_warning("unable to determine map definition size " 658 "section %s, %d maps in %zd bytes\n", 659 obj->path, nr_maps, data->d_size); 660 return -EINVAL; 661 } 662 663 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 664 if (!obj->maps) { 665 pr_warning("alloc maps for object failed\n"); 666 return -ENOMEM; 667 } 668 obj->nr_maps = nr_maps; 669 670 /* 671 * fill all fd with -1 so won't close incorrect 672 * fd (fd=0 is stdin) when failure (zclose won't close 673 * negative fd)). 674 */ 675 for (i = 0; i < nr_maps; i++) 676 obj->maps[i].fd = -1; 677 678 /* 679 * Fill obj->maps using data in "maps" section. 680 */ 681 for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 682 GElf_Sym sym; 683 const char *map_name; 684 struct bpf_map_def *def; 685 686 if (!gelf_getsym(symbols, i, &sym)) 687 continue; 688 if (sym.st_shndx != obj->efile.maps_shndx) 689 continue; 690 691 map_name = elf_strptr(obj->efile.elf, 692 obj->efile.strtabidx, 693 sym.st_name); 694 obj->maps[map_idx].offset = sym.st_value; 695 if (sym.st_value + map_def_sz > data->d_size) { 696 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 697 obj->path, map_name); 698 return -EINVAL; 699 } 700 701 obj->maps[map_idx].name = strdup(map_name); 702 if (!obj->maps[map_idx].name) { 703 pr_warning("failed to alloc map name\n"); 704 return -ENOMEM; 705 } 706 pr_debug("map %d is \"%s\"\n", map_idx, 707 obj->maps[map_idx].name); 708 def = (struct bpf_map_def *)(data->d_buf + sym.st_value); 709 /* 710 * If the definition of the map in the object file fits in 711 * bpf_map_def, copy it. Any extra fields in our version 712 * of bpf_map_def will default to zero as a result of the 713 * calloc above. 714 */ 715 if (map_def_sz <= sizeof(struct bpf_map_def)) { 716 memcpy(&obj->maps[map_idx].def, def, map_def_sz); 717 } else { 718 /* 719 * Here the map structure being read is bigger than what 720 * we expect, truncate if the excess bits are all zero. 721 * If they are not zero, reject this map as 722 * incompatible. 723 */ 724 char *b; 725 for (b = ((char *)def) + sizeof(struct bpf_map_def); 726 b < ((char *)def) + map_def_sz; b++) { 727 if (*b != 0) { 728 pr_warning("maps section in %s: \"%s\" " 729 "has unrecognized, non-zero " 730 "options\n", 731 obj->path, map_name); 732 return -EINVAL; 733 } 734 } 735 memcpy(&obj->maps[map_idx].def, def, 736 sizeof(struct bpf_map_def)); 737 } 738 map_idx++; 739 } 740 741 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); 742 return 0; 743 } 744 745 static int bpf_object__elf_collect(struct bpf_object *obj) 746 { 747 Elf *elf = obj->efile.elf; 748 GElf_Ehdr *ep = &obj->efile.ehdr; 749 Elf_Scn *scn = NULL; 750 int idx = 0, err = 0; 751 752 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 753 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 754 pr_warning("failed to get e_shstrndx from %s\n", 755 obj->path); 756 return -LIBBPF_ERRNO__FORMAT; 757 } 758 759 while ((scn = elf_nextscn(elf, scn)) != NULL) { 760 char *name; 761 GElf_Shdr sh; 762 Elf_Data *data; 763 764 idx++; 765 if (gelf_getshdr(scn, &sh) != &sh) { 766 pr_warning("failed to get section header from %s\n", 767 obj->path); 768 err = -LIBBPF_ERRNO__FORMAT; 769 goto out; 770 } 771 772 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 773 if (!name) { 774 pr_warning("failed to get section name from %s\n", 775 obj->path); 776 err = -LIBBPF_ERRNO__FORMAT; 777 goto out; 778 } 779 780 data = elf_getdata(scn, 0); 781 if (!data) { 782 pr_warning("failed to get section data from %s(%s)\n", 783 name, obj->path); 784 err = -LIBBPF_ERRNO__FORMAT; 785 goto out; 786 } 787 pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n", 788 name, (unsigned long)data->d_size, 789 (int)sh.sh_link, (unsigned long)sh.sh_flags, 790 (int)sh.sh_type); 791 792 if (strcmp(name, "license") == 0) 793 err = bpf_object__init_license(obj, 794 data->d_buf, 795 data->d_size); 796 else if (strcmp(name, "version") == 0) 797 err = bpf_object__init_kversion(obj, 798 data->d_buf, 799 data->d_size); 800 else if (strcmp(name, "maps") == 0) 801 obj->efile.maps_shndx = idx; 802 else if (sh.sh_type == SHT_SYMTAB) { 803 if (obj->efile.symbols) { 804 pr_warning("bpf: multiple SYMTAB in %s\n", 805 obj->path); 806 err = -LIBBPF_ERRNO__FORMAT; 807 } else { 808 obj->efile.symbols = data; 809 obj->efile.strtabidx = sh.sh_link; 810 } 811 } else if ((sh.sh_type == SHT_PROGBITS) && 812 (sh.sh_flags & SHF_EXECINSTR) && 813 (data->d_size > 0)) { 814 if (strcmp(name, ".text") == 0) 815 obj->efile.text_shndx = idx; 816 err = bpf_object__add_program(obj, data->d_buf, 817 data->d_size, name, idx); 818 if (err) { 819 char errmsg[STRERR_BUFSIZE]; 820 821 strerror_r(-err, errmsg, sizeof(errmsg)); 822 pr_warning("failed to alloc program %s (%s): %s", 823 name, obj->path, errmsg); 824 } 825 } else if (sh.sh_type == SHT_REL) { 826 void *reloc = obj->efile.reloc; 827 int nr_reloc = obj->efile.nr_reloc + 1; 828 829 reloc = realloc(reloc, 830 sizeof(*obj->efile.reloc) * nr_reloc); 831 if (!reloc) { 832 pr_warning("realloc failed\n"); 833 err = -ENOMEM; 834 } else { 835 int n = nr_reloc - 1; 836 837 obj->efile.reloc = reloc; 838 obj->efile.nr_reloc = nr_reloc; 839 840 obj->efile.reloc[n].shdr = sh; 841 obj->efile.reloc[n].data = data; 842 } 843 } 844 if (err) 845 goto out; 846 } 847 848 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 849 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 850 return LIBBPF_ERRNO__FORMAT; 851 } 852 if (obj->efile.maps_shndx >= 0) { 853 err = bpf_object__init_maps(obj); 854 if (err) 855 goto out; 856 } 857 err = bpf_object__init_prog_names(obj); 858 out: 859 return err; 860 } 861 862 static struct bpf_program * 863 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 864 { 865 struct bpf_program *prog; 866 size_t i; 867 868 for (i = 0; i < obj->nr_programs; i++) { 869 prog = &obj->programs[i]; 870 if (prog->idx == idx) 871 return prog; 872 } 873 return NULL; 874 } 875 876 static int 877 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 878 Elf_Data *data, struct bpf_object *obj) 879 { 880 Elf_Data *symbols = obj->efile.symbols; 881 int text_shndx = obj->efile.text_shndx; 882 int maps_shndx = obj->efile.maps_shndx; 883 struct bpf_map *maps = obj->maps; 884 size_t nr_maps = obj->nr_maps; 885 int i, nrels; 886 887 pr_debug("collecting relocating info for: '%s'\n", 888 prog->section_name); 889 nrels = shdr->sh_size / shdr->sh_entsize; 890 891 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 892 if (!prog->reloc_desc) { 893 pr_warning("failed to alloc memory in relocation\n"); 894 return -ENOMEM; 895 } 896 prog->nr_reloc = nrels; 897 898 for (i = 0; i < nrels; i++) { 899 GElf_Sym sym; 900 GElf_Rel rel; 901 unsigned int insn_idx; 902 struct bpf_insn *insns = prog->insns; 903 size_t map_idx; 904 905 if (!gelf_getrel(data, i, &rel)) { 906 pr_warning("relocation: failed to get %d reloc\n", i); 907 return -LIBBPF_ERRNO__FORMAT; 908 } 909 910 if (!gelf_getsym(symbols, 911 GELF_R_SYM(rel.r_info), 912 &sym)) { 913 pr_warning("relocation: symbol %"PRIx64" not found\n", 914 GELF_R_SYM(rel.r_info)); 915 return -LIBBPF_ERRNO__FORMAT; 916 } 917 pr_debug("relo for %lld value %lld name %d\n", 918 (long long) (rel.r_info >> 32), 919 (long long) sym.st_value, sym.st_name); 920 921 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) { 922 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", 923 prog->section_name, sym.st_shndx); 924 return -LIBBPF_ERRNO__RELOC; 925 } 926 927 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 928 pr_debug("relocation: insn_idx=%u\n", insn_idx); 929 930 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { 931 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { 932 pr_warning("incorrect bpf_call opcode\n"); 933 return -LIBBPF_ERRNO__RELOC; 934 } 935 prog->reloc_desc[i].type = RELO_CALL; 936 prog->reloc_desc[i].insn_idx = insn_idx; 937 prog->reloc_desc[i].text_off = sym.st_value; 938 continue; 939 } 940 941 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 942 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 943 insn_idx, insns[insn_idx].code); 944 return -LIBBPF_ERRNO__RELOC; 945 } 946 947 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */ 948 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 949 if (maps[map_idx].offset == sym.st_value) { 950 pr_debug("relocation: find map %zd (%s) for insn %u\n", 951 map_idx, maps[map_idx].name, insn_idx); 952 break; 953 } 954 } 955 956 if (map_idx >= nr_maps) { 957 pr_warning("bpf relocation: map_idx %d large than %d\n", 958 (int)map_idx, (int)nr_maps - 1); 959 return -LIBBPF_ERRNO__RELOC; 960 } 961 962 prog->reloc_desc[i].type = RELO_LD64; 963 prog->reloc_desc[i].insn_idx = insn_idx; 964 prog->reloc_desc[i].map_idx = map_idx; 965 } 966 return 0; 967 } 968 969 static int 970 bpf_object__create_maps(struct bpf_object *obj) 971 { 972 unsigned int i; 973 974 for (i = 0; i < obj->nr_maps; i++) { 975 struct bpf_map_def *def = &obj->maps[i].def; 976 int *pfd = &obj->maps[i].fd; 977 978 *pfd = bpf_create_map_name(def->type, 979 obj->maps[i].name, 980 def->key_size, 981 def->value_size, 982 def->max_entries, 983 def->map_flags); 984 if (*pfd < 0) { 985 size_t j; 986 int err = *pfd; 987 988 pr_warning("failed to create map (name: '%s'): %s\n", 989 obj->maps[i].name, 990 strerror(errno)); 991 for (j = 0; j < i; j++) 992 zclose(obj->maps[j].fd); 993 return err; 994 } 995 pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd); 996 } 997 998 return 0; 999 } 1000 1001 static int 1002 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, 1003 struct reloc_desc *relo) 1004 { 1005 struct bpf_insn *insn, *new_insn; 1006 struct bpf_program *text; 1007 size_t new_cnt; 1008 1009 if (relo->type != RELO_CALL) 1010 return -LIBBPF_ERRNO__RELOC; 1011 1012 if (prog->idx == obj->efile.text_shndx) { 1013 pr_warning("relo in .text insn %d into off %d\n", 1014 relo->insn_idx, relo->text_off); 1015 return -LIBBPF_ERRNO__RELOC; 1016 } 1017 1018 if (prog->main_prog_cnt == 0) { 1019 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); 1020 if (!text) { 1021 pr_warning("no .text section found yet relo into text exist\n"); 1022 return -LIBBPF_ERRNO__RELOC; 1023 } 1024 new_cnt = prog->insns_cnt + text->insns_cnt; 1025 new_insn = realloc(prog->insns, new_cnt * sizeof(*insn)); 1026 if (!new_insn) { 1027 pr_warning("oom in prog realloc\n"); 1028 return -ENOMEM; 1029 } 1030 memcpy(new_insn + prog->insns_cnt, text->insns, 1031 text->insns_cnt * sizeof(*insn)); 1032 prog->insns = new_insn; 1033 prog->main_prog_cnt = prog->insns_cnt; 1034 prog->insns_cnt = new_cnt; 1035 } 1036 insn = &prog->insns[relo->insn_idx]; 1037 insn->imm += prog->main_prog_cnt - relo->insn_idx; 1038 pr_debug("added %zd insn from %s to prog %s\n", 1039 text->insns_cnt, text->section_name, prog->section_name); 1040 return 0; 1041 } 1042 1043 static int 1044 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1045 { 1046 int i, err; 1047 1048 if (!prog || !prog->reloc_desc) 1049 return 0; 1050 1051 for (i = 0; i < prog->nr_reloc; i++) { 1052 if (prog->reloc_desc[i].type == RELO_LD64) { 1053 struct bpf_insn *insns = prog->insns; 1054 int insn_idx, map_idx; 1055 1056 insn_idx = prog->reloc_desc[i].insn_idx; 1057 map_idx = prog->reloc_desc[i].map_idx; 1058 1059 if (insn_idx >= (int)prog->insns_cnt) { 1060 pr_warning("relocation out of range: '%s'\n", 1061 prog->section_name); 1062 return -LIBBPF_ERRNO__RELOC; 1063 } 1064 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1065 insns[insn_idx].imm = obj->maps[map_idx].fd; 1066 } else { 1067 err = bpf_program__reloc_text(prog, obj, 1068 &prog->reloc_desc[i]); 1069 if (err) 1070 return err; 1071 } 1072 } 1073 1074 zfree(&prog->reloc_desc); 1075 prog->nr_reloc = 0; 1076 return 0; 1077 } 1078 1079 1080 static int 1081 bpf_object__relocate(struct bpf_object *obj) 1082 { 1083 struct bpf_program *prog; 1084 size_t i; 1085 int err; 1086 1087 for (i = 0; i < obj->nr_programs; i++) { 1088 prog = &obj->programs[i]; 1089 1090 err = bpf_program__relocate(prog, obj); 1091 if (err) { 1092 pr_warning("failed to relocate '%s'\n", 1093 prog->section_name); 1094 return err; 1095 } 1096 } 1097 return 0; 1098 } 1099 1100 static int bpf_object__collect_reloc(struct bpf_object *obj) 1101 { 1102 int i, err; 1103 1104 if (!obj_elf_valid(obj)) { 1105 pr_warning("Internal error: elf object is closed\n"); 1106 return -LIBBPF_ERRNO__INTERNAL; 1107 } 1108 1109 for (i = 0; i < obj->efile.nr_reloc; i++) { 1110 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 1111 Elf_Data *data = obj->efile.reloc[i].data; 1112 int idx = shdr->sh_info; 1113 struct bpf_program *prog; 1114 1115 if (shdr->sh_type != SHT_REL) { 1116 pr_warning("internal error at %d\n", __LINE__); 1117 return -LIBBPF_ERRNO__INTERNAL; 1118 } 1119 1120 prog = bpf_object__find_prog_by_idx(obj, idx); 1121 if (!prog) { 1122 pr_warning("relocation failed: no %d section\n", 1123 idx); 1124 return -LIBBPF_ERRNO__RELOC; 1125 } 1126 1127 err = bpf_program__collect_reloc(prog, 1128 shdr, data, 1129 obj); 1130 if (err) 1131 return err; 1132 } 1133 return 0; 1134 } 1135 1136 static int 1137 load_program(enum bpf_prog_type type, const char *name, struct bpf_insn *insns, 1138 int insns_cnt, char *license, u32 kern_version, int *pfd) 1139 { 1140 int ret; 1141 char *log_buf; 1142 1143 if (!insns || !insns_cnt) 1144 return -EINVAL; 1145 1146 log_buf = malloc(BPF_LOG_BUF_SIZE); 1147 if (!log_buf) 1148 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 1149 1150 ret = bpf_load_program_name(type, name, insns, insns_cnt, license, 1151 kern_version, log_buf, BPF_LOG_BUF_SIZE); 1152 1153 if (ret >= 0) { 1154 *pfd = ret; 1155 ret = 0; 1156 goto out; 1157 } 1158 1159 ret = -LIBBPF_ERRNO__LOAD; 1160 pr_warning("load bpf program failed: %s\n", strerror(errno)); 1161 1162 if (log_buf && log_buf[0] != '\0') { 1163 ret = -LIBBPF_ERRNO__VERIFY; 1164 pr_warning("-- BEGIN DUMP LOG ---\n"); 1165 pr_warning("\n%s\n", log_buf); 1166 pr_warning("-- END LOG --\n"); 1167 } else if (insns_cnt >= BPF_MAXINSNS) { 1168 pr_warning("Program too large (%d insns), at most %d insns\n", 1169 insns_cnt, BPF_MAXINSNS); 1170 ret = -LIBBPF_ERRNO__PROG2BIG; 1171 } else { 1172 /* Wrong program type? */ 1173 if (type != BPF_PROG_TYPE_KPROBE) { 1174 int fd; 1175 1176 fd = bpf_load_program_name(BPF_PROG_TYPE_KPROBE, name, 1177 insns, insns_cnt, license, 1178 kern_version, NULL, 0); 1179 if (fd >= 0) { 1180 close(fd); 1181 ret = -LIBBPF_ERRNO__PROGTYPE; 1182 goto out; 1183 } 1184 } 1185 1186 if (log_buf) 1187 ret = -LIBBPF_ERRNO__KVER; 1188 } 1189 1190 out: 1191 free(log_buf); 1192 return ret; 1193 } 1194 1195 static int 1196 bpf_program__load(struct bpf_program *prog, 1197 char *license, u32 kern_version) 1198 { 1199 int err = 0, fd, i; 1200 1201 if (prog->instances.nr < 0 || !prog->instances.fds) { 1202 if (prog->preprocessor) { 1203 pr_warning("Internal error: can't load program '%s'\n", 1204 prog->section_name); 1205 return -LIBBPF_ERRNO__INTERNAL; 1206 } 1207 1208 prog->instances.fds = malloc(sizeof(int)); 1209 if (!prog->instances.fds) { 1210 pr_warning("Not enough memory for BPF fds\n"); 1211 return -ENOMEM; 1212 } 1213 prog->instances.nr = 1; 1214 prog->instances.fds[0] = -1; 1215 } 1216 1217 if (!prog->preprocessor) { 1218 if (prog->instances.nr != 1) { 1219 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 1220 prog->section_name, prog->instances.nr); 1221 } 1222 err = load_program(prog->type, prog->name, prog->insns, 1223 prog->insns_cnt, license, kern_version, &fd); 1224 if (!err) 1225 prog->instances.fds[0] = fd; 1226 goto out; 1227 } 1228 1229 for (i = 0; i < prog->instances.nr; i++) { 1230 struct bpf_prog_prep_result result; 1231 bpf_program_prep_t preprocessor = prog->preprocessor; 1232 1233 bzero(&result, sizeof(result)); 1234 err = preprocessor(prog, i, prog->insns, 1235 prog->insns_cnt, &result); 1236 if (err) { 1237 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 1238 i, prog->section_name); 1239 goto out; 1240 } 1241 1242 if (!result.new_insn_ptr || !result.new_insn_cnt) { 1243 pr_debug("Skip loading the %dth instance of program '%s'\n", 1244 i, prog->section_name); 1245 prog->instances.fds[i] = -1; 1246 if (result.pfd) 1247 *result.pfd = -1; 1248 continue; 1249 } 1250 1251 err = load_program(prog->type, prog->name, 1252 result.new_insn_ptr, 1253 result.new_insn_cnt, 1254 license, kern_version, &fd); 1255 1256 if (err) { 1257 pr_warning("Loading the %dth instance of program '%s' failed\n", 1258 i, prog->section_name); 1259 goto out; 1260 } 1261 1262 if (result.pfd) 1263 *result.pfd = fd; 1264 prog->instances.fds[i] = fd; 1265 } 1266 out: 1267 if (err) 1268 pr_warning("failed to load program '%s'\n", 1269 prog->section_name); 1270 zfree(&prog->insns); 1271 prog->insns_cnt = 0; 1272 return err; 1273 } 1274 1275 static int 1276 bpf_object__load_progs(struct bpf_object *obj) 1277 { 1278 size_t i; 1279 int err; 1280 1281 for (i = 0; i < obj->nr_programs; i++) { 1282 if (obj->programs[i].idx == obj->efile.text_shndx) 1283 continue; 1284 err = bpf_program__load(&obj->programs[i], 1285 obj->license, 1286 obj->kern_version); 1287 if (err) 1288 return err; 1289 } 1290 return 0; 1291 } 1292 1293 static int bpf_object__validate(struct bpf_object *obj) 1294 { 1295 if (obj->kern_version == 0) { 1296 pr_warning("%s doesn't provide kernel version\n", 1297 obj->path); 1298 return -LIBBPF_ERRNO__KVERSION; 1299 } 1300 return 0; 1301 } 1302 1303 static struct bpf_object * 1304 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) 1305 { 1306 struct bpf_object *obj; 1307 int err; 1308 1309 if (elf_version(EV_CURRENT) == EV_NONE) { 1310 pr_warning("failed to init libelf for %s\n", path); 1311 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 1312 } 1313 1314 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 1315 if (IS_ERR(obj)) 1316 return obj; 1317 1318 CHECK_ERR(bpf_object__elf_init(obj), err, out); 1319 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 1320 CHECK_ERR(bpf_object__elf_collect(obj), err, out); 1321 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 1322 CHECK_ERR(bpf_object__validate(obj), err, out); 1323 1324 bpf_object__elf_finish(obj); 1325 return obj; 1326 out: 1327 bpf_object__close(obj); 1328 return ERR_PTR(err); 1329 } 1330 1331 struct bpf_object *bpf_object__open(const char *path) 1332 { 1333 /* param validation */ 1334 if (!path) 1335 return NULL; 1336 1337 pr_debug("loading %s\n", path); 1338 1339 return __bpf_object__open(path, NULL, 0); 1340 } 1341 1342 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 1343 size_t obj_buf_sz, 1344 const char *name) 1345 { 1346 char tmp_name[64]; 1347 1348 /* param validation */ 1349 if (!obj_buf || obj_buf_sz <= 0) 1350 return NULL; 1351 1352 if (!name) { 1353 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 1354 (unsigned long)obj_buf, 1355 (unsigned long)obj_buf_sz); 1356 tmp_name[sizeof(tmp_name) - 1] = '\0'; 1357 name = tmp_name; 1358 } 1359 pr_debug("loading object '%s' from buffer\n", 1360 name); 1361 1362 return __bpf_object__open(name, obj_buf, obj_buf_sz); 1363 } 1364 1365 int bpf_object__unload(struct bpf_object *obj) 1366 { 1367 size_t i; 1368 1369 if (!obj) 1370 return -EINVAL; 1371 1372 for (i = 0; i < obj->nr_maps; i++) 1373 zclose(obj->maps[i].fd); 1374 1375 for (i = 0; i < obj->nr_programs; i++) 1376 bpf_program__unload(&obj->programs[i]); 1377 1378 return 0; 1379 } 1380 1381 int bpf_object__load(struct bpf_object *obj) 1382 { 1383 int err; 1384 1385 if (!obj) 1386 return -EINVAL; 1387 1388 if (obj->loaded) { 1389 pr_warning("object should not be loaded twice\n"); 1390 return -EINVAL; 1391 } 1392 1393 obj->loaded = true; 1394 1395 CHECK_ERR(bpf_object__create_maps(obj), err, out); 1396 CHECK_ERR(bpf_object__relocate(obj), err, out); 1397 CHECK_ERR(bpf_object__load_progs(obj), err, out); 1398 1399 return 0; 1400 out: 1401 bpf_object__unload(obj); 1402 pr_warning("failed to load object '%s'\n", obj->path); 1403 return err; 1404 } 1405 1406 static int check_path(const char *path) 1407 { 1408 struct statfs st_fs; 1409 char *dname, *dir; 1410 int err = 0; 1411 1412 if (path == NULL) 1413 return -EINVAL; 1414 1415 dname = strdup(path); 1416 if (dname == NULL) 1417 return -ENOMEM; 1418 1419 dir = dirname(dname); 1420 if (statfs(dir, &st_fs)) { 1421 pr_warning("failed to statfs %s: %s\n", dir, strerror(errno)); 1422 err = -errno; 1423 } 1424 free(dname); 1425 1426 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 1427 pr_warning("specified path %s is not on BPF FS\n", path); 1428 err = -EINVAL; 1429 } 1430 1431 return err; 1432 } 1433 1434 int bpf_program__pin_instance(struct bpf_program *prog, const char *path, 1435 int instance) 1436 { 1437 int err; 1438 1439 err = check_path(path); 1440 if (err) 1441 return err; 1442 1443 if (prog == NULL) { 1444 pr_warning("invalid program pointer\n"); 1445 return -EINVAL; 1446 } 1447 1448 if (instance < 0 || instance >= prog->instances.nr) { 1449 pr_warning("invalid prog instance %d of prog %s (max %d)\n", 1450 instance, prog->section_name, prog->instances.nr); 1451 return -EINVAL; 1452 } 1453 1454 if (bpf_obj_pin(prog->instances.fds[instance], path)) { 1455 pr_warning("failed to pin program: %s\n", strerror(errno)); 1456 return -errno; 1457 } 1458 pr_debug("pinned program '%s'\n", path); 1459 1460 return 0; 1461 } 1462 1463 static int make_dir(const char *path) 1464 { 1465 int err = 0; 1466 1467 if (mkdir(path, 0700) && errno != EEXIST) 1468 err = -errno; 1469 1470 if (err) 1471 pr_warning("failed to mkdir %s: %s\n", path, strerror(-err)); 1472 return err; 1473 } 1474 1475 int bpf_program__pin(struct bpf_program *prog, const char *path) 1476 { 1477 int i, err; 1478 1479 err = check_path(path); 1480 if (err) 1481 return err; 1482 1483 if (prog == NULL) { 1484 pr_warning("invalid program pointer\n"); 1485 return -EINVAL; 1486 } 1487 1488 if (prog->instances.nr <= 0) { 1489 pr_warning("no instances of prog %s to pin\n", 1490 prog->section_name); 1491 return -EINVAL; 1492 } 1493 1494 err = make_dir(path); 1495 if (err) 1496 return err; 1497 1498 for (i = 0; i < prog->instances.nr; i++) { 1499 char buf[PATH_MAX]; 1500 int len; 1501 1502 len = snprintf(buf, PATH_MAX, "%s/%d", path, i); 1503 if (len < 0) 1504 return -EINVAL; 1505 else if (len >= PATH_MAX) 1506 return -ENAMETOOLONG; 1507 1508 err = bpf_program__pin_instance(prog, buf, i); 1509 if (err) 1510 return err; 1511 } 1512 1513 return 0; 1514 } 1515 1516 int bpf_map__pin(struct bpf_map *map, const char *path) 1517 { 1518 int err; 1519 1520 err = check_path(path); 1521 if (err) 1522 return err; 1523 1524 if (map == NULL) { 1525 pr_warning("invalid map pointer\n"); 1526 return -EINVAL; 1527 } 1528 1529 if (bpf_obj_pin(map->fd, path)) { 1530 pr_warning("failed to pin map: %s\n", strerror(errno)); 1531 return -errno; 1532 } 1533 1534 pr_debug("pinned map '%s'\n", path); 1535 return 0; 1536 } 1537 1538 int bpf_object__pin(struct bpf_object *obj, const char *path) 1539 { 1540 struct bpf_program *prog; 1541 struct bpf_map *map; 1542 int err; 1543 1544 if (!obj) 1545 return -ENOENT; 1546 1547 if (!obj->loaded) { 1548 pr_warning("object not yet loaded; load it first\n"); 1549 return -ENOENT; 1550 } 1551 1552 err = make_dir(path); 1553 if (err) 1554 return err; 1555 1556 bpf_map__for_each(map, obj) { 1557 char buf[PATH_MAX]; 1558 int len; 1559 1560 len = snprintf(buf, PATH_MAX, "%s/%s", path, 1561 bpf_map__name(map)); 1562 if (len < 0) 1563 return -EINVAL; 1564 else if (len >= PATH_MAX) 1565 return -ENAMETOOLONG; 1566 1567 err = bpf_map__pin(map, buf); 1568 if (err) 1569 return err; 1570 } 1571 1572 bpf_object__for_each_program(prog, obj) { 1573 char buf[PATH_MAX]; 1574 int len; 1575 1576 len = snprintf(buf, PATH_MAX, "%s/%s", path, 1577 prog->section_name); 1578 if (len < 0) 1579 return -EINVAL; 1580 else if (len >= PATH_MAX) 1581 return -ENAMETOOLONG; 1582 1583 err = bpf_program__pin(prog, buf); 1584 if (err) 1585 return err; 1586 } 1587 1588 return 0; 1589 } 1590 1591 void bpf_object__close(struct bpf_object *obj) 1592 { 1593 size_t i; 1594 1595 if (!obj) 1596 return; 1597 1598 if (obj->clear_priv) 1599 obj->clear_priv(obj, obj->priv); 1600 1601 bpf_object__elf_finish(obj); 1602 bpf_object__unload(obj); 1603 1604 for (i = 0; i < obj->nr_maps; i++) { 1605 zfree(&obj->maps[i].name); 1606 if (obj->maps[i].clear_priv) 1607 obj->maps[i].clear_priv(&obj->maps[i], 1608 obj->maps[i].priv); 1609 obj->maps[i].priv = NULL; 1610 obj->maps[i].clear_priv = NULL; 1611 } 1612 zfree(&obj->maps); 1613 obj->nr_maps = 0; 1614 1615 if (obj->programs && obj->nr_programs) { 1616 for (i = 0; i < obj->nr_programs; i++) 1617 bpf_program__exit(&obj->programs[i]); 1618 } 1619 zfree(&obj->programs); 1620 1621 list_del(&obj->list); 1622 free(obj); 1623 } 1624 1625 struct bpf_object * 1626 bpf_object__next(struct bpf_object *prev) 1627 { 1628 struct bpf_object *next; 1629 1630 if (!prev) 1631 next = list_first_entry(&bpf_objects_list, 1632 struct bpf_object, 1633 list); 1634 else 1635 next = list_next_entry(prev, list); 1636 1637 /* Empty list is noticed here so don't need checking on entry. */ 1638 if (&next->list == &bpf_objects_list) 1639 return NULL; 1640 1641 return next; 1642 } 1643 1644 const char *bpf_object__name(struct bpf_object *obj) 1645 { 1646 return obj ? obj->path : ERR_PTR(-EINVAL); 1647 } 1648 1649 unsigned int bpf_object__kversion(struct bpf_object *obj) 1650 { 1651 return obj ? obj->kern_version : 0; 1652 } 1653 1654 int bpf_object__set_priv(struct bpf_object *obj, void *priv, 1655 bpf_object_clear_priv_t clear_priv) 1656 { 1657 if (obj->priv && obj->clear_priv) 1658 obj->clear_priv(obj, obj->priv); 1659 1660 obj->priv = priv; 1661 obj->clear_priv = clear_priv; 1662 return 0; 1663 } 1664 1665 void *bpf_object__priv(struct bpf_object *obj) 1666 { 1667 return obj ? obj->priv : ERR_PTR(-EINVAL); 1668 } 1669 1670 struct bpf_program * 1671 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 1672 { 1673 size_t idx; 1674 1675 if (!obj->programs) 1676 return NULL; 1677 /* First handler */ 1678 if (prev == NULL) 1679 return &obj->programs[0]; 1680 1681 if (prev->obj != obj) { 1682 pr_warning("error: program handler doesn't match object\n"); 1683 return NULL; 1684 } 1685 1686 idx = (prev - obj->programs) + 1; 1687 if (idx >= obj->nr_programs) 1688 return NULL; 1689 return &obj->programs[idx]; 1690 } 1691 1692 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 1693 bpf_program_clear_priv_t clear_priv) 1694 { 1695 if (prog->priv && prog->clear_priv) 1696 prog->clear_priv(prog, prog->priv); 1697 1698 prog->priv = priv; 1699 prog->clear_priv = clear_priv; 1700 return 0; 1701 } 1702 1703 void *bpf_program__priv(struct bpf_program *prog) 1704 { 1705 return prog ? prog->priv : ERR_PTR(-EINVAL); 1706 } 1707 1708 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 1709 { 1710 const char *title; 1711 1712 title = prog->section_name; 1713 if (needs_copy) { 1714 title = strdup(title); 1715 if (!title) { 1716 pr_warning("failed to strdup program title\n"); 1717 return ERR_PTR(-ENOMEM); 1718 } 1719 } 1720 1721 return title; 1722 } 1723 1724 int bpf_program__fd(struct bpf_program *prog) 1725 { 1726 return bpf_program__nth_fd(prog, 0); 1727 } 1728 1729 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 1730 bpf_program_prep_t prep) 1731 { 1732 int *instances_fds; 1733 1734 if (nr_instances <= 0 || !prep) 1735 return -EINVAL; 1736 1737 if (prog->instances.nr > 0 || prog->instances.fds) { 1738 pr_warning("Can't set pre-processor after loading\n"); 1739 return -EINVAL; 1740 } 1741 1742 instances_fds = malloc(sizeof(int) * nr_instances); 1743 if (!instances_fds) { 1744 pr_warning("alloc memory failed for fds\n"); 1745 return -ENOMEM; 1746 } 1747 1748 /* fill all fd with -1 */ 1749 memset(instances_fds, -1, sizeof(int) * nr_instances); 1750 1751 prog->instances.nr = nr_instances; 1752 prog->instances.fds = instances_fds; 1753 prog->preprocessor = prep; 1754 return 0; 1755 } 1756 1757 int bpf_program__nth_fd(struct bpf_program *prog, int n) 1758 { 1759 int fd; 1760 1761 if (n >= prog->instances.nr || n < 0) { 1762 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 1763 n, prog->section_name, prog->instances.nr); 1764 return -EINVAL; 1765 } 1766 1767 fd = prog->instances.fds[n]; 1768 if (fd < 0) { 1769 pr_warning("%dth instance of program '%s' is invalid\n", 1770 n, prog->section_name); 1771 return -ENOENT; 1772 } 1773 1774 return fd; 1775 } 1776 1777 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 1778 { 1779 prog->type = type; 1780 } 1781 1782 static bool bpf_program__is_type(struct bpf_program *prog, 1783 enum bpf_prog_type type) 1784 { 1785 return prog ? (prog->type == type) : false; 1786 } 1787 1788 #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ 1789 int bpf_program__set_##NAME(struct bpf_program *prog) \ 1790 { \ 1791 if (!prog) \ 1792 return -EINVAL; \ 1793 bpf_program__set_type(prog, TYPE); \ 1794 return 0; \ 1795 } \ 1796 \ 1797 bool bpf_program__is_##NAME(struct bpf_program *prog) \ 1798 { \ 1799 return bpf_program__is_type(prog, TYPE); \ 1800 } \ 1801 1802 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); 1803 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); 1804 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); 1805 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); 1806 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); 1807 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 1808 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 1809 1810 #define BPF_PROG_SEC(string, type) { string, sizeof(string) - 1, type } 1811 static const struct { 1812 const char *sec; 1813 size_t len; 1814 enum bpf_prog_type prog_type; 1815 } section_names[] = { 1816 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), 1817 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), 1818 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), 1819 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), 1820 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), 1821 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), 1822 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), 1823 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), 1824 BPF_PROG_SEC("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), 1825 BPF_PROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK), 1826 BPF_PROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE), 1827 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), 1828 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), 1829 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), 1830 BPF_PROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS), 1831 BPF_PROG_SEC("sk_skb", BPF_PROG_TYPE_SK_SKB), 1832 }; 1833 #undef BPF_PROG_SEC 1834 1835 static enum bpf_prog_type bpf_program__guess_type(struct bpf_program *prog) 1836 { 1837 int i; 1838 1839 if (!prog->section_name) 1840 goto err; 1841 1842 for (i = 0; i < ARRAY_SIZE(section_names); i++) 1843 if (strncmp(prog->section_name, section_names[i].sec, 1844 section_names[i].len) == 0) 1845 return section_names[i].prog_type; 1846 1847 err: 1848 pr_warning("failed to guess program type based on section name %s\n", 1849 prog->section_name); 1850 1851 return BPF_PROG_TYPE_UNSPEC; 1852 } 1853 1854 int bpf_map__fd(struct bpf_map *map) 1855 { 1856 return map ? map->fd : -EINVAL; 1857 } 1858 1859 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 1860 { 1861 return map ? &map->def : ERR_PTR(-EINVAL); 1862 } 1863 1864 const char *bpf_map__name(struct bpf_map *map) 1865 { 1866 return map ? map->name : NULL; 1867 } 1868 1869 int bpf_map__set_priv(struct bpf_map *map, void *priv, 1870 bpf_map_clear_priv_t clear_priv) 1871 { 1872 if (!map) 1873 return -EINVAL; 1874 1875 if (map->priv) { 1876 if (map->clear_priv) 1877 map->clear_priv(map, map->priv); 1878 } 1879 1880 map->priv = priv; 1881 map->clear_priv = clear_priv; 1882 return 0; 1883 } 1884 1885 void *bpf_map__priv(struct bpf_map *map) 1886 { 1887 return map ? map->priv : ERR_PTR(-EINVAL); 1888 } 1889 1890 struct bpf_map * 1891 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 1892 { 1893 size_t idx; 1894 struct bpf_map *s, *e; 1895 1896 if (!obj || !obj->maps) 1897 return NULL; 1898 1899 s = obj->maps; 1900 e = obj->maps + obj->nr_maps; 1901 1902 if (prev == NULL) 1903 return s; 1904 1905 if ((prev < s) || (prev >= e)) { 1906 pr_warning("error in %s: map handler doesn't belong to object\n", 1907 __func__); 1908 return NULL; 1909 } 1910 1911 idx = (prev - obj->maps) + 1; 1912 if (idx >= obj->nr_maps) 1913 return NULL; 1914 return &obj->maps[idx]; 1915 } 1916 1917 struct bpf_map * 1918 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 1919 { 1920 struct bpf_map *pos; 1921 1922 bpf_map__for_each(pos, obj) { 1923 if (pos->name && !strcmp(pos->name, name)) 1924 return pos; 1925 } 1926 return NULL; 1927 } 1928 1929 struct bpf_map * 1930 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) 1931 { 1932 int i; 1933 1934 for (i = 0; i < obj->nr_maps; i++) { 1935 if (obj->maps[i].offset == offset) 1936 return &obj->maps[i]; 1937 } 1938 return ERR_PTR(-ENOENT); 1939 } 1940 1941 long libbpf_get_error(const void *ptr) 1942 { 1943 if (IS_ERR(ptr)) 1944 return PTR_ERR(ptr); 1945 return 0; 1946 } 1947 1948 int bpf_prog_load(const char *file, enum bpf_prog_type type, 1949 struct bpf_object **pobj, int *prog_fd) 1950 { 1951 struct bpf_program *prog, *first_prog = NULL; 1952 struct bpf_object *obj; 1953 int err; 1954 1955 obj = bpf_object__open(file); 1956 if (IS_ERR(obj)) 1957 return -ENOENT; 1958 1959 bpf_object__for_each_program(prog, obj) { 1960 /* 1961 * If type is not specified, try to guess it based on 1962 * section name. 1963 */ 1964 if (type == BPF_PROG_TYPE_UNSPEC) { 1965 type = bpf_program__guess_type(prog); 1966 if (type == BPF_PROG_TYPE_UNSPEC) { 1967 bpf_object__close(obj); 1968 return -EINVAL; 1969 } 1970 } 1971 1972 bpf_program__set_type(prog, type); 1973 if (prog->idx != obj->efile.text_shndx && !first_prog) 1974 first_prog = prog; 1975 } 1976 1977 if (!first_prog) { 1978 pr_warning("object file doesn't contain bpf program\n"); 1979 bpf_object__close(obj); 1980 return -ENOENT; 1981 } 1982 1983 err = bpf_object__load(obj); 1984 if (err) { 1985 bpf_object__close(obj); 1986 return -EINVAL; 1987 } 1988 1989 *pobj = obj; 1990 *prog_fd = bpf_program__fd(first_prog); 1991 return 0; 1992 } 1993