1 /* 2 * Common eBPF ELF object loading operations. 3 * 4 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 5 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 6 * Copyright (C) 2015 Huawei Inc. 7 * 8 * This program is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU Lesser General Public 10 * License as published by the Free Software Foundation; 11 * version 2.1 of the License (not later!) 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU Lesser General Public License for more details. 17 * 18 * You should have received a copy of the GNU Lesser General Public 19 * License along with this program; if not, see <http://www.gnu.org/licenses> 20 */ 21 22 #include <stdlib.h> 23 #include <stdio.h> 24 #include <stdarg.h> 25 #include <inttypes.h> 26 #include <string.h> 27 #include <unistd.h> 28 #include <fcntl.h> 29 #include <errno.h> 30 #include <asm/unistd.h> 31 #include <linux/kernel.h> 32 #include <linux/bpf.h> 33 #include <linux/list.h> 34 #include <libelf.h> 35 #include <gelf.h> 36 37 #include "libbpf.h" 38 #include "bpf.h" 39 40 #define __printf(a, b) __attribute__((format(printf, a, b))) 41 42 __printf(1, 2) 43 static int __base_pr(const char *format, ...) 44 { 45 va_list args; 46 int err; 47 48 va_start(args, format); 49 err = vfprintf(stderr, format, args); 50 va_end(args); 51 return err; 52 } 53 54 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; 55 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; 56 static __printf(1, 2) libbpf_print_fn_t __pr_debug; 57 58 #define __pr(func, fmt, ...) \ 59 do { \ 60 if ((func)) \ 61 (func)("libbpf: " fmt, ##__VA_ARGS__); \ 62 } while (0) 63 64 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) 65 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) 66 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) 67 68 void libbpf_set_print(libbpf_print_fn_t warn, 69 libbpf_print_fn_t info, 70 libbpf_print_fn_t debug) 71 { 72 __pr_warning = warn; 73 __pr_info = info; 74 __pr_debug = debug; 75 } 76 77 #define STRERR_BUFSIZE 128 78 79 #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) 80 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) 81 #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) 82 83 static const char *libbpf_strerror_table[NR_ERRNO] = { 84 [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", 85 [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", 86 [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", 87 [ERRCODE_OFFSET(ENDIAN)] = "Endian mismatch", 88 [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", 89 [ERRCODE_OFFSET(RELOC)] = "Relocation failed", 90 [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", 91 [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", 92 [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", 93 }; 94 95 int libbpf_strerror(int err, char *buf, size_t size) 96 { 97 if (!buf || !size) 98 return -1; 99 100 err = err > 0 ? err : -err; 101 102 if (err < __LIBBPF_ERRNO__START) { 103 int ret; 104 105 ret = strerror_r(err, buf, size); 106 buf[size - 1] = '\0'; 107 return ret; 108 } 109 110 if (err < __LIBBPF_ERRNO__END) { 111 const char *msg; 112 113 msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; 114 snprintf(buf, size, "%s", msg); 115 buf[size - 1] = '\0'; 116 return 0; 117 } 118 119 snprintf(buf, size, "Unknown libbpf error %d", err); 120 buf[size - 1] = '\0'; 121 return -1; 122 } 123 124 #define CHECK_ERR(action, err, out) do { \ 125 err = action; \ 126 if (err) \ 127 goto out; \ 128 } while(0) 129 130 131 /* Copied from tools/perf/util/util.h */ 132 #ifndef zfree 133 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 134 #endif 135 136 #ifndef zclose 137 # define zclose(fd) ({ \ 138 int ___err = 0; \ 139 if ((fd) >= 0) \ 140 ___err = close((fd)); \ 141 fd = -1; \ 142 ___err; }) 143 #endif 144 145 #ifdef HAVE_LIBELF_MMAP_SUPPORT 146 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP 147 #else 148 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ 149 #endif 150 151 /* 152 * bpf_prog should be a better name but it has been used in 153 * linux/filter.h. 154 */ 155 struct bpf_program { 156 /* Index in elf obj file, for relocation use. */ 157 int idx; 158 char *section_name; 159 struct bpf_insn *insns; 160 size_t insns_cnt; 161 162 struct { 163 int insn_idx; 164 int map_idx; 165 } *reloc_desc; 166 int nr_reloc; 167 168 struct { 169 int nr; 170 int *fds; 171 } instances; 172 bpf_program_prep_t preprocessor; 173 174 struct bpf_object *obj; 175 void *priv; 176 bpf_program_clear_priv_t clear_priv; 177 }; 178 179 struct bpf_map { 180 int fd; 181 char *name; 182 struct bpf_map_def def; 183 void *priv; 184 bpf_map_clear_priv_t clear_priv; 185 }; 186 187 static LIST_HEAD(bpf_objects_list); 188 189 struct bpf_object { 190 char license[64]; 191 u32 kern_version; 192 193 struct bpf_program *programs; 194 size_t nr_programs; 195 struct bpf_map *maps; 196 size_t nr_maps; 197 198 bool loaded; 199 200 /* 201 * Information when doing elf related work. Only valid if fd 202 * is valid. 203 */ 204 struct { 205 int fd; 206 void *obj_buf; 207 size_t obj_buf_sz; 208 Elf *elf; 209 GElf_Ehdr ehdr; 210 Elf_Data *symbols; 211 size_t strtabidx; 212 struct { 213 GElf_Shdr shdr; 214 Elf_Data *data; 215 } *reloc; 216 int nr_reloc; 217 int maps_shndx; 218 } efile; 219 /* 220 * All loaded bpf_object is linked in a list, which is 221 * hidden to caller. bpf_objects__<func> handlers deal with 222 * all objects. 223 */ 224 struct list_head list; 225 char path[]; 226 }; 227 #define obj_elf_valid(o) ((o)->efile.elf) 228 229 static void bpf_program__unload(struct bpf_program *prog) 230 { 231 int i; 232 233 if (!prog) 234 return; 235 236 /* 237 * If the object is opened but the program was never loaded, 238 * it is possible that prog->instances.nr == -1. 239 */ 240 if (prog->instances.nr > 0) { 241 for (i = 0; i < prog->instances.nr; i++) 242 zclose(prog->instances.fds[i]); 243 } else if (prog->instances.nr != -1) { 244 pr_warning("Internal error: instances.nr is %d\n", 245 prog->instances.nr); 246 } 247 248 prog->instances.nr = -1; 249 zfree(&prog->instances.fds); 250 } 251 252 static void bpf_program__exit(struct bpf_program *prog) 253 { 254 if (!prog) 255 return; 256 257 if (prog->clear_priv) 258 prog->clear_priv(prog, prog->priv); 259 260 prog->priv = NULL; 261 prog->clear_priv = NULL; 262 263 bpf_program__unload(prog); 264 zfree(&prog->section_name); 265 zfree(&prog->insns); 266 zfree(&prog->reloc_desc); 267 268 prog->nr_reloc = 0; 269 prog->insns_cnt = 0; 270 prog->idx = -1; 271 } 272 273 static int 274 bpf_program__init(void *data, size_t size, char *name, int idx, 275 struct bpf_program *prog) 276 { 277 if (size < sizeof(struct bpf_insn)) { 278 pr_warning("corrupted section '%s'\n", name); 279 return -EINVAL; 280 } 281 282 bzero(prog, sizeof(*prog)); 283 284 prog->section_name = strdup(name); 285 if (!prog->section_name) { 286 pr_warning("failed to alloc name for prog %s\n", 287 name); 288 goto errout; 289 } 290 291 prog->insns = malloc(size); 292 if (!prog->insns) { 293 pr_warning("failed to alloc insns for %s\n", name); 294 goto errout; 295 } 296 prog->insns_cnt = size / sizeof(struct bpf_insn); 297 memcpy(prog->insns, data, 298 prog->insns_cnt * sizeof(struct bpf_insn)); 299 prog->idx = idx; 300 prog->instances.fds = NULL; 301 prog->instances.nr = -1; 302 303 return 0; 304 errout: 305 bpf_program__exit(prog); 306 return -ENOMEM; 307 } 308 309 static int 310 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, 311 char *name, int idx) 312 { 313 struct bpf_program prog, *progs; 314 int nr_progs, err; 315 316 err = bpf_program__init(data, size, name, idx, &prog); 317 if (err) 318 return err; 319 320 progs = obj->programs; 321 nr_progs = obj->nr_programs; 322 323 progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); 324 if (!progs) { 325 /* 326 * In this case the original obj->programs 327 * is still valid, so don't need special treat for 328 * bpf_close_object(). 329 */ 330 pr_warning("failed to alloc a new program '%s'\n", 331 name); 332 bpf_program__exit(&prog); 333 return -ENOMEM; 334 } 335 336 pr_debug("found program %s\n", prog.section_name); 337 obj->programs = progs; 338 obj->nr_programs = nr_progs + 1; 339 prog.obj = obj; 340 progs[nr_progs] = prog; 341 return 0; 342 } 343 344 static struct bpf_object *bpf_object__new(const char *path, 345 void *obj_buf, 346 size_t obj_buf_sz) 347 { 348 struct bpf_object *obj; 349 350 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 351 if (!obj) { 352 pr_warning("alloc memory failed for %s\n", path); 353 return ERR_PTR(-ENOMEM); 354 } 355 356 strcpy(obj->path, path); 357 obj->efile.fd = -1; 358 359 /* 360 * Caller of this function should also calls 361 * bpf_object__elf_finish() after data collection to return 362 * obj_buf to user. If not, we should duplicate the buffer to 363 * avoid user freeing them before elf finish. 364 */ 365 obj->efile.obj_buf = obj_buf; 366 obj->efile.obj_buf_sz = obj_buf_sz; 367 obj->efile.maps_shndx = -1; 368 369 obj->loaded = false; 370 371 INIT_LIST_HEAD(&obj->list); 372 list_add(&obj->list, &bpf_objects_list); 373 return obj; 374 } 375 376 static void bpf_object__elf_finish(struct bpf_object *obj) 377 { 378 if (!obj_elf_valid(obj)) 379 return; 380 381 if (obj->efile.elf) { 382 elf_end(obj->efile.elf); 383 obj->efile.elf = NULL; 384 } 385 obj->efile.symbols = NULL; 386 387 zfree(&obj->efile.reloc); 388 obj->efile.nr_reloc = 0; 389 zclose(obj->efile.fd); 390 obj->efile.obj_buf = NULL; 391 obj->efile.obj_buf_sz = 0; 392 } 393 394 static int bpf_object__elf_init(struct bpf_object *obj) 395 { 396 int err = 0; 397 GElf_Ehdr *ep; 398 399 if (obj_elf_valid(obj)) { 400 pr_warning("elf init: internal error\n"); 401 return -LIBBPF_ERRNO__LIBELF; 402 } 403 404 if (obj->efile.obj_buf_sz > 0) { 405 /* 406 * obj_buf should have been validated by 407 * bpf_object__open_buffer(). 408 */ 409 obj->efile.elf = elf_memory(obj->efile.obj_buf, 410 obj->efile.obj_buf_sz); 411 } else { 412 obj->efile.fd = open(obj->path, O_RDONLY); 413 if (obj->efile.fd < 0) { 414 pr_warning("failed to open %s: %s\n", obj->path, 415 strerror(errno)); 416 return -errno; 417 } 418 419 obj->efile.elf = elf_begin(obj->efile.fd, 420 LIBBPF_ELF_C_READ_MMAP, 421 NULL); 422 } 423 424 if (!obj->efile.elf) { 425 pr_warning("failed to open %s as ELF file\n", 426 obj->path); 427 err = -LIBBPF_ERRNO__LIBELF; 428 goto errout; 429 } 430 431 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { 432 pr_warning("failed to get EHDR from %s\n", 433 obj->path); 434 err = -LIBBPF_ERRNO__FORMAT; 435 goto errout; 436 } 437 ep = &obj->efile.ehdr; 438 439 if ((ep->e_type != ET_REL) || (ep->e_machine != 0)) { 440 pr_warning("%s is not an eBPF object file\n", 441 obj->path); 442 err = -LIBBPF_ERRNO__FORMAT; 443 goto errout; 444 } 445 446 return 0; 447 errout: 448 bpf_object__elf_finish(obj); 449 return err; 450 } 451 452 static int 453 bpf_object__check_endianness(struct bpf_object *obj) 454 { 455 static unsigned int const endian = 1; 456 457 switch (obj->efile.ehdr.e_ident[EI_DATA]) { 458 case ELFDATA2LSB: 459 /* We are big endian, BPF obj is little endian. */ 460 if (*(unsigned char const *)&endian != 1) 461 goto mismatch; 462 break; 463 464 case ELFDATA2MSB: 465 /* We are little endian, BPF obj is big endian. */ 466 if (*(unsigned char const *)&endian != 0) 467 goto mismatch; 468 break; 469 default: 470 return -LIBBPF_ERRNO__ENDIAN; 471 } 472 473 return 0; 474 475 mismatch: 476 pr_warning("Error: endianness mismatch.\n"); 477 return -LIBBPF_ERRNO__ENDIAN; 478 } 479 480 static int 481 bpf_object__init_license(struct bpf_object *obj, 482 void *data, size_t size) 483 { 484 memcpy(obj->license, data, 485 min(size, sizeof(obj->license) - 1)); 486 pr_debug("license of %s is %s\n", obj->path, obj->license); 487 return 0; 488 } 489 490 static int 491 bpf_object__init_kversion(struct bpf_object *obj, 492 void *data, size_t size) 493 { 494 u32 kver; 495 496 if (size != sizeof(kver)) { 497 pr_warning("invalid kver section in %s\n", obj->path); 498 return -LIBBPF_ERRNO__FORMAT; 499 } 500 memcpy(&kver, data, sizeof(kver)); 501 obj->kern_version = kver; 502 pr_debug("kernel version of %s is %x\n", obj->path, 503 obj->kern_version); 504 return 0; 505 } 506 507 static int 508 bpf_object__init_maps(struct bpf_object *obj, void *data, 509 size_t size) 510 { 511 size_t nr_maps; 512 int i; 513 514 nr_maps = size / sizeof(struct bpf_map_def); 515 if (!data || !nr_maps) { 516 pr_debug("%s doesn't need map definition\n", 517 obj->path); 518 return 0; 519 } 520 521 pr_debug("maps in %s: %zd bytes\n", obj->path, size); 522 523 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 524 if (!obj->maps) { 525 pr_warning("alloc maps for object failed\n"); 526 return -ENOMEM; 527 } 528 obj->nr_maps = nr_maps; 529 530 for (i = 0; i < nr_maps; i++) { 531 struct bpf_map_def *def = &obj->maps[i].def; 532 533 /* 534 * fill all fd with -1 so won't close incorrect 535 * fd (fd=0 is stdin) when failure (zclose won't close 536 * negative fd)). 537 */ 538 obj->maps[i].fd = -1; 539 540 /* Save map definition into obj->maps */ 541 *def = ((struct bpf_map_def *)data)[i]; 542 } 543 return 0; 544 } 545 546 static int 547 bpf_object__init_maps_name(struct bpf_object *obj) 548 { 549 int i; 550 Elf_Data *symbols = obj->efile.symbols; 551 552 if (!symbols || obj->efile.maps_shndx < 0) 553 return -EINVAL; 554 555 for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 556 GElf_Sym sym; 557 size_t map_idx; 558 const char *map_name; 559 560 if (!gelf_getsym(symbols, i, &sym)) 561 continue; 562 if (sym.st_shndx != obj->efile.maps_shndx) 563 continue; 564 565 map_name = elf_strptr(obj->efile.elf, 566 obj->efile.strtabidx, 567 sym.st_name); 568 map_idx = sym.st_value / sizeof(struct bpf_map_def); 569 if (map_idx >= obj->nr_maps) { 570 pr_warning("index of map \"%s\" is buggy: %zu > %zu\n", 571 map_name, map_idx, obj->nr_maps); 572 continue; 573 } 574 obj->maps[map_idx].name = strdup(map_name); 575 if (!obj->maps[map_idx].name) { 576 pr_warning("failed to alloc map name\n"); 577 return -ENOMEM; 578 } 579 pr_debug("map %zu is \"%s\"\n", map_idx, 580 obj->maps[map_idx].name); 581 } 582 return 0; 583 } 584 585 static int bpf_object__elf_collect(struct bpf_object *obj) 586 { 587 Elf *elf = obj->efile.elf; 588 GElf_Ehdr *ep = &obj->efile.ehdr; 589 Elf_Scn *scn = NULL; 590 int idx = 0, err = 0; 591 592 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 593 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { 594 pr_warning("failed to get e_shstrndx from %s\n", 595 obj->path); 596 return -LIBBPF_ERRNO__FORMAT; 597 } 598 599 while ((scn = elf_nextscn(elf, scn)) != NULL) { 600 char *name; 601 GElf_Shdr sh; 602 Elf_Data *data; 603 604 idx++; 605 if (gelf_getshdr(scn, &sh) != &sh) { 606 pr_warning("failed to get section header from %s\n", 607 obj->path); 608 err = -LIBBPF_ERRNO__FORMAT; 609 goto out; 610 } 611 612 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); 613 if (!name) { 614 pr_warning("failed to get section name from %s\n", 615 obj->path); 616 err = -LIBBPF_ERRNO__FORMAT; 617 goto out; 618 } 619 620 data = elf_getdata(scn, 0); 621 if (!data) { 622 pr_warning("failed to get section data from %s(%s)\n", 623 name, obj->path); 624 err = -LIBBPF_ERRNO__FORMAT; 625 goto out; 626 } 627 pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n", 628 name, (unsigned long)data->d_size, 629 (int)sh.sh_link, (unsigned long)sh.sh_flags, 630 (int)sh.sh_type); 631 632 if (strcmp(name, "license") == 0) 633 err = bpf_object__init_license(obj, 634 data->d_buf, 635 data->d_size); 636 else if (strcmp(name, "version") == 0) 637 err = bpf_object__init_kversion(obj, 638 data->d_buf, 639 data->d_size); 640 else if (strcmp(name, "maps") == 0) { 641 err = bpf_object__init_maps(obj, data->d_buf, 642 data->d_size); 643 obj->efile.maps_shndx = idx; 644 } else if (sh.sh_type == SHT_SYMTAB) { 645 if (obj->efile.symbols) { 646 pr_warning("bpf: multiple SYMTAB in %s\n", 647 obj->path); 648 err = -LIBBPF_ERRNO__FORMAT; 649 } else { 650 obj->efile.symbols = data; 651 obj->efile.strtabidx = sh.sh_link; 652 } 653 } else if ((sh.sh_type == SHT_PROGBITS) && 654 (sh.sh_flags & SHF_EXECINSTR) && 655 (data->d_size > 0)) { 656 err = bpf_object__add_program(obj, data->d_buf, 657 data->d_size, name, idx); 658 if (err) { 659 char errmsg[STRERR_BUFSIZE]; 660 661 strerror_r(-err, errmsg, sizeof(errmsg)); 662 pr_warning("failed to alloc program %s (%s): %s", 663 name, obj->path, errmsg); 664 } 665 } else if (sh.sh_type == SHT_REL) { 666 void *reloc = obj->efile.reloc; 667 int nr_reloc = obj->efile.nr_reloc + 1; 668 669 reloc = realloc(reloc, 670 sizeof(*obj->efile.reloc) * nr_reloc); 671 if (!reloc) { 672 pr_warning("realloc failed\n"); 673 err = -ENOMEM; 674 } else { 675 int n = nr_reloc - 1; 676 677 obj->efile.reloc = reloc; 678 obj->efile.nr_reloc = nr_reloc; 679 680 obj->efile.reloc[n].shdr = sh; 681 obj->efile.reloc[n].data = data; 682 } 683 } 684 if (err) 685 goto out; 686 } 687 688 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { 689 pr_warning("Corrupted ELF file: index of strtab invalid\n"); 690 return LIBBPF_ERRNO__FORMAT; 691 } 692 if (obj->efile.maps_shndx >= 0) 693 err = bpf_object__init_maps_name(obj); 694 out: 695 return err; 696 } 697 698 static struct bpf_program * 699 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) 700 { 701 struct bpf_program *prog; 702 size_t i; 703 704 for (i = 0; i < obj->nr_programs; i++) { 705 prog = &obj->programs[i]; 706 if (prog->idx == idx) 707 return prog; 708 } 709 return NULL; 710 } 711 712 static int 713 bpf_program__collect_reloc(struct bpf_program *prog, 714 size_t nr_maps, GElf_Shdr *shdr, 715 Elf_Data *data, Elf_Data *symbols, 716 int maps_shndx) 717 { 718 int i, nrels; 719 720 pr_debug("collecting relocating info for: '%s'\n", 721 prog->section_name); 722 nrels = shdr->sh_size / shdr->sh_entsize; 723 724 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); 725 if (!prog->reloc_desc) { 726 pr_warning("failed to alloc memory in relocation\n"); 727 return -ENOMEM; 728 } 729 prog->nr_reloc = nrels; 730 731 for (i = 0; i < nrels; i++) { 732 GElf_Sym sym; 733 GElf_Rel rel; 734 unsigned int insn_idx; 735 struct bpf_insn *insns = prog->insns; 736 size_t map_idx; 737 738 if (!gelf_getrel(data, i, &rel)) { 739 pr_warning("relocation: failed to get %d reloc\n", i); 740 return -LIBBPF_ERRNO__FORMAT; 741 } 742 743 if (!gelf_getsym(symbols, 744 GELF_R_SYM(rel.r_info), 745 &sym)) { 746 pr_warning("relocation: symbol %"PRIx64" not found\n", 747 GELF_R_SYM(rel.r_info)); 748 return -LIBBPF_ERRNO__FORMAT; 749 } 750 751 if (sym.st_shndx != maps_shndx) { 752 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", 753 prog->section_name, sym.st_shndx); 754 return -LIBBPF_ERRNO__RELOC; 755 } 756 757 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 758 pr_debug("relocation: insn_idx=%u\n", insn_idx); 759 760 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 761 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 762 insn_idx, insns[insn_idx].code); 763 return -LIBBPF_ERRNO__RELOC; 764 } 765 766 map_idx = sym.st_value / sizeof(struct bpf_map_def); 767 if (map_idx >= nr_maps) { 768 pr_warning("bpf relocation: map_idx %d large than %d\n", 769 (int)map_idx, (int)nr_maps - 1); 770 return -LIBBPF_ERRNO__RELOC; 771 } 772 773 prog->reloc_desc[i].insn_idx = insn_idx; 774 prog->reloc_desc[i].map_idx = map_idx; 775 } 776 return 0; 777 } 778 779 static int 780 bpf_object__create_maps(struct bpf_object *obj) 781 { 782 unsigned int i; 783 784 for (i = 0; i < obj->nr_maps; i++) { 785 struct bpf_map_def *def = &obj->maps[i].def; 786 int *pfd = &obj->maps[i].fd; 787 788 *pfd = bpf_create_map(def->type, 789 def->key_size, 790 def->value_size, 791 def->max_entries); 792 if (*pfd < 0) { 793 size_t j; 794 int err = *pfd; 795 796 pr_warning("failed to create map: %s\n", 797 strerror(errno)); 798 for (j = 0; j < i; j++) 799 zclose(obj->maps[j].fd); 800 return err; 801 } 802 pr_debug("create map: fd=%d\n", *pfd); 803 } 804 805 return 0; 806 } 807 808 static int 809 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 810 { 811 int i; 812 813 if (!prog || !prog->reloc_desc) 814 return 0; 815 816 for (i = 0; i < prog->nr_reloc; i++) { 817 int insn_idx, map_idx; 818 struct bpf_insn *insns = prog->insns; 819 820 insn_idx = prog->reloc_desc[i].insn_idx; 821 map_idx = prog->reloc_desc[i].map_idx; 822 823 if (insn_idx >= (int)prog->insns_cnt) { 824 pr_warning("relocation out of range: '%s'\n", 825 prog->section_name); 826 return -LIBBPF_ERRNO__RELOC; 827 } 828 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 829 insns[insn_idx].imm = obj->maps[map_idx].fd; 830 } 831 832 zfree(&prog->reloc_desc); 833 prog->nr_reloc = 0; 834 return 0; 835 } 836 837 838 static int 839 bpf_object__relocate(struct bpf_object *obj) 840 { 841 struct bpf_program *prog; 842 size_t i; 843 int err; 844 845 for (i = 0; i < obj->nr_programs; i++) { 846 prog = &obj->programs[i]; 847 848 err = bpf_program__relocate(prog, obj); 849 if (err) { 850 pr_warning("failed to relocate '%s'\n", 851 prog->section_name); 852 return err; 853 } 854 } 855 return 0; 856 } 857 858 static int bpf_object__collect_reloc(struct bpf_object *obj) 859 { 860 int i, err; 861 862 if (!obj_elf_valid(obj)) { 863 pr_warning("Internal error: elf object is closed\n"); 864 return -LIBBPF_ERRNO__INTERNAL; 865 } 866 867 for (i = 0; i < obj->efile.nr_reloc; i++) { 868 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; 869 Elf_Data *data = obj->efile.reloc[i].data; 870 int idx = shdr->sh_info; 871 struct bpf_program *prog; 872 size_t nr_maps = obj->nr_maps; 873 874 if (shdr->sh_type != SHT_REL) { 875 pr_warning("internal error at %d\n", __LINE__); 876 return -LIBBPF_ERRNO__INTERNAL; 877 } 878 879 prog = bpf_object__find_prog_by_idx(obj, idx); 880 if (!prog) { 881 pr_warning("relocation failed: no %d section\n", 882 idx); 883 return -LIBBPF_ERRNO__RELOC; 884 } 885 886 err = bpf_program__collect_reloc(prog, nr_maps, 887 shdr, data, 888 obj->efile.symbols, 889 obj->efile.maps_shndx); 890 if (err) 891 return err; 892 } 893 return 0; 894 } 895 896 static int 897 load_program(struct bpf_insn *insns, int insns_cnt, 898 char *license, u32 kern_version, int *pfd) 899 { 900 int ret; 901 char *log_buf; 902 903 if (!insns || !insns_cnt) 904 return -EINVAL; 905 906 log_buf = malloc(BPF_LOG_BUF_SIZE); 907 if (!log_buf) 908 pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); 909 910 ret = bpf_load_program(BPF_PROG_TYPE_KPROBE, insns, 911 insns_cnt, license, kern_version, 912 log_buf, BPF_LOG_BUF_SIZE); 913 914 if (ret >= 0) { 915 *pfd = ret; 916 ret = 0; 917 goto out; 918 } 919 920 ret = -LIBBPF_ERRNO__LOAD; 921 pr_warning("load bpf program failed: %s\n", strerror(errno)); 922 923 if (log_buf && log_buf[0] != '\0') { 924 ret = -LIBBPF_ERRNO__VERIFY; 925 pr_warning("-- BEGIN DUMP LOG ---\n"); 926 pr_warning("\n%s\n", log_buf); 927 pr_warning("-- END LOG --\n"); 928 } else { 929 if (insns_cnt >= BPF_MAXINSNS) { 930 pr_warning("Program too large (%d insns), at most %d insns\n", 931 insns_cnt, BPF_MAXINSNS); 932 ret = -LIBBPF_ERRNO__PROG2BIG; 933 } else if (log_buf) { 934 pr_warning("log buffer is empty\n"); 935 ret = -LIBBPF_ERRNO__KVER; 936 } 937 } 938 939 out: 940 free(log_buf); 941 return ret; 942 } 943 944 static int 945 bpf_program__load(struct bpf_program *prog, 946 char *license, u32 kern_version) 947 { 948 int err = 0, fd, i; 949 950 if (prog->instances.nr < 0 || !prog->instances.fds) { 951 if (prog->preprocessor) { 952 pr_warning("Internal error: can't load program '%s'\n", 953 prog->section_name); 954 return -LIBBPF_ERRNO__INTERNAL; 955 } 956 957 prog->instances.fds = malloc(sizeof(int)); 958 if (!prog->instances.fds) { 959 pr_warning("Not enough memory for BPF fds\n"); 960 return -ENOMEM; 961 } 962 prog->instances.nr = 1; 963 prog->instances.fds[0] = -1; 964 } 965 966 if (!prog->preprocessor) { 967 if (prog->instances.nr != 1) { 968 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", 969 prog->section_name, prog->instances.nr); 970 } 971 err = load_program(prog->insns, prog->insns_cnt, 972 license, kern_version, &fd); 973 if (!err) 974 prog->instances.fds[0] = fd; 975 goto out; 976 } 977 978 for (i = 0; i < prog->instances.nr; i++) { 979 struct bpf_prog_prep_result result; 980 bpf_program_prep_t preprocessor = prog->preprocessor; 981 982 bzero(&result, sizeof(result)); 983 err = preprocessor(prog, i, prog->insns, 984 prog->insns_cnt, &result); 985 if (err) { 986 pr_warning("Preprocessing the %dth instance of program '%s' failed\n", 987 i, prog->section_name); 988 goto out; 989 } 990 991 if (!result.new_insn_ptr || !result.new_insn_cnt) { 992 pr_debug("Skip loading the %dth instance of program '%s'\n", 993 i, prog->section_name); 994 prog->instances.fds[i] = -1; 995 if (result.pfd) 996 *result.pfd = -1; 997 continue; 998 } 999 1000 err = load_program(result.new_insn_ptr, 1001 result.new_insn_cnt, 1002 license, kern_version, &fd); 1003 1004 if (err) { 1005 pr_warning("Loading the %dth instance of program '%s' failed\n", 1006 i, prog->section_name); 1007 goto out; 1008 } 1009 1010 if (result.pfd) 1011 *result.pfd = fd; 1012 prog->instances.fds[i] = fd; 1013 } 1014 out: 1015 if (err) 1016 pr_warning("failed to load program '%s'\n", 1017 prog->section_name); 1018 zfree(&prog->insns); 1019 prog->insns_cnt = 0; 1020 return err; 1021 } 1022 1023 static int 1024 bpf_object__load_progs(struct bpf_object *obj) 1025 { 1026 size_t i; 1027 int err; 1028 1029 for (i = 0; i < obj->nr_programs; i++) { 1030 err = bpf_program__load(&obj->programs[i], 1031 obj->license, 1032 obj->kern_version); 1033 if (err) 1034 return err; 1035 } 1036 return 0; 1037 } 1038 1039 static int bpf_object__validate(struct bpf_object *obj) 1040 { 1041 if (obj->kern_version == 0) { 1042 pr_warning("%s doesn't provide kernel version\n", 1043 obj->path); 1044 return -LIBBPF_ERRNO__KVERSION; 1045 } 1046 return 0; 1047 } 1048 1049 static struct bpf_object * 1050 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) 1051 { 1052 struct bpf_object *obj; 1053 int err; 1054 1055 if (elf_version(EV_CURRENT) == EV_NONE) { 1056 pr_warning("failed to init libelf for %s\n", path); 1057 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 1058 } 1059 1060 obj = bpf_object__new(path, obj_buf, obj_buf_sz); 1061 if (IS_ERR(obj)) 1062 return obj; 1063 1064 CHECK_ERR(bpf_object__elf_init(obj), err, out); 1065 CHECK_ERR(bpf_object__check_endianness(obj), err, out); 1066 CHECK_ERR(bpf_object__elf_collect(obj), err, out); 1067 CHECK_ERR(bpf_object__collect_reloc(obj), err, out); 1068 CHECK_ERR(bpf_object__validate(obj), err, out); 1069 1070 bpf_object__elf_finish(obj); 1071 return obj; 1072 out: 1073 bpf_object__close(obj); 1074 return ERR_PTR(err); 1075 } 1076 1077 struct bpf_object *bpf_object__open(const char *path) 1078 { 1079 /* param validation */ 1080 if (!path) 1081 return NULL; 1082 1083 pr_debug("loading %s\n", path); 1084 1085 return __bpf_object__open(path, NULL, 0); 1086 } 1087 1088 struct bpf_object *bpf_object__open_buffer(void *obj_buf, 1089 size_t obj_buf_sz, 1090 const char *name) 1091 { 1092 char tmp_name[64]; 1093 1094 /* param validation */ 1095 if (!obj_buf || obj_buf_sz <= 0) 1096 return NULL; 1097 1098 if (!name) { 1099 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 1100 (unsigned long)obj_buf, 1101 (unsigned long)obj_buf_sz); 1102 tmp_name[sizeof(tmp_name) - 1] = '\0'; 1103 name = tmp_name; 1104 } 1105 pr_debug("loading object '%s' from buffer\n", 1106 name); 1107 1108 return __bpf_object__open(name, obj_buf, obj_buf_sz); 1109 } 1110 1111 int bpf_object__unload(struct bpf_object *obj) 1112 { 1113 size_t i; 1114 1115 if (!obj) 1116 return -EINVAL; 1117 1118 for (i = 0; i < obj->nr_maps; i++) 1119 zclose(obj->maps[i].fd); 1120 1121 for (i = 0; i < obj->nr_programs; i++) 1122 bpf_program__unload(&obj->programs[i]); 1123 1124 return 0; 1125 } 1126 1127 int bpf_object__load(struct bpf_object *obj) 1128 { 1129 int err; 1130 1131 if (!obj) 1132 return -EINVAL; 1133 1134 if (obj->loaded) { 1135 pr_warning("object should not be loaded twice\n"); 1136 return -EINVAL; 1137 } 1138 1139 obj->loaded = true; 1140 1141 CHECK_ERR(bpf_object__create_maps(obj), err, out); 1142 CHECK_ERR(bpf_object__relocate(obj), err, out); 1143 CHECK_ERR(bpf_object__load_progs(obj), err, out); 1144 1145 return 0; 1146 out: 1147 bpf_object__unload(obj); 1148 pr_warning("failed to load object '%s'\n", obj->path); 1149 return err; 1150 } 1151 1152 void bpf_object__close(struct bpf_object *obj) 1153 { 1154 size_t i; 1155 1156 if (!obj) 1157 return; 1158 1159 bpf_object__elf_finish(obj); 1160 bpf_object__unload(obj); 1161 1162 for (i = 0; i < obj->nr_maps; i++) { 1163 zfree(&obj->maps[i].name); 1164 if (obj->maps[i].clear_priv) 1165 obj->maps[i].clear_priv(&obj->maps[i], 1166 obj->maps[i].priv); 1167 obj->maps[i].priv = NULL; 1168 obj->maps[i].clear_priv = NULL; 1169 } 1170 zfree(&obj->maps); 1171 obj->nr_maps = 0; 1172 1173 if (obj->programs && obj->nr_programs) { 1174 for (i = 0; i < obj->nr_programs; i++) 1175 bpf_program__exit(&obj->programs[i]); 1176 } 1177 zfree(&obj->programs); 1178 1179 list_del(&obj->list); 1180 free(obj); 1181 } 1182 1183 struct bpf_object * 1184 bpf_object__next(struct bpf_object *prev) 1185 { 1186 struct bpf_object *next; 1187 1188 if (!prev) 1189 next = list_first_entry(&bpf_objects_list, 1190 struct bpf_object, 1191 list); 1192 else 1193 next = list_next_entry(prev, list); 1194 1195 /* Empty list is noticed here so don't need checking on entry. */ 1196 if (&next->list == &bpf_objects_list) 1197 return NULL; 1198 1199 return next; 1200 } 1201 1202 const char *bpf_object__name(struct bpf_object *obj) 1203 { 1204 return obj ? obj->path : ERR_PTR(-EINVAL); 1205 } 1206 1207 unsigned int bpf_object__kversion(struct bpf_object *obj) 1208 { 1209 return obj ? obj->kern_version : 0; 1210 } 1211 1212 struct bpf_program * 1213 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) 1214 { 1215 size_t idx; 1216 1217 if (!obj->programs) 1218 return NULL; 1219 /* First handler */ 1220 if (prev == NULL) 1221 return &obj->programs[0]; 1222 1223 if (prev->obj != obj) { 1224 pr_warning("error: program handler doesn't match object\n"); 1225 return NULL; 1226 } 1227 1228 idx = (prev - obj->programs) + 1; 1229 if (idx >= obj->nr_programs) 1230 return NULL; 1231 return &obj->programs[idx]; 1232 } 1233 1234 int bpf_program__set_priv(struct bpf_program *prog, void *priv, 1235 bpf_program_clear_priv_t clear_priv) 1236 { 1237 if (prog->priv && prog->clear_priv) 1238 prog->clear_priv(prog, prog->priv); 1239 1240 prog->priv = priv; 1241 prog->clear_priv = clear_priv; 1242 return 0; 1243 } 1244 1245 void *bpf_program__priv(struct bpf_program *prog) 1246 { 1247 return prog ? prog->priv : ERR_PTR(-EINVAL); 1248 } 1249 1250 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) 1251 { 1252 const char *title; 1253 1254 title = prog->section_name; 1255 if (needs_copy) { 1256 title = strdup(title); 1257 if (!title) { 1258 pr_warning("failed to strdup program title\n"); 1259 return ERR_PTR(-ENOMEM); 1260 } 1261 } 1262 1263 return title; 1264 } 1265 1266 int bpf_program__fd(struct bpf_program *prog) 1267 { 1268 return bpf_program__nth_fd(prog, 0); 1269 } 1270 1271 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, 1272 bpf_program_prep_t prep) 1273 { 1274 int *instances_fds; 1275 1276 if (nr_instances <= 0 || !prep) 1277 return -EINVAL; 1278 1279 if (prog->instances.nr > 0 || prog->instances.fds) { 1280 pr_warning("Can't set pre-processor after loading\n"); 1281 return -EINVAL; 1282 } 1283 1284 instances_fds = malloc(sizeof(int) * nr_instances); 1285 if (!instances_fds) { 1286 pr_warning("alloc memory failed for fds\n"); 1287 return -ENOMEM; 1288 } 1289 1290 /* fill all fd with -1 */ 1291 memset(instances_fds, -1, sizeof(int) * nr_instances); 1292 1293 prog->instances.nr = nr_instances; 1294 prog->instances.fds = instances_fds; 1295 prog->preprocessor = prep; 1296 return 0; 1297 } 1298 1299 int bpf_program__nth_fd(struct bpf_program *prog, int n) 1300 { 1301 int fd; 1302 1303 if (n >= prog->instances.nr || n < 0) { 1304 pr_warning("Can't get the %dth fd from program %s: only %d instances\n", 1305 n, prog->section_name, prog->instances.nr); 1306 return -EINVAL; 1307 } 1308 1309 fd = prog->instances.fds[n]; 1310 if (fd < 0) { 1311 pr_warning("%dth instance of program '%s' is invalid\n", 1312 n, prog->section_name); 1313 return -ENOENT; 1314 } 1315 1316 return fd; 1317 } 1318 1319 int bpf_map__fd(struct bpf_map *map) 1320 { 1321 return map ? map->fd : -EINVAL; 1322 } 1323 1324 const struct bpf_map_def *bpf_map__def(struct bpf_map *map) 1325 { 1326 return map ? &map->def : ERR_PTR(-EINVAL); 1327 } 1328 1329 const char *bpf_map__name(struct bpf_map *map) 1330 { 1331 return map ? map->name : NULL; 1332 } 1333 1334 int bpf_map__set_priv(struct bpf_map *map, void *priv, 1335 bpf_map_clear_priv_t clear_priv) 1336 { 1337 if (!map) 1338 return -EINVAL; 1339 1340 if (map->priv) { 1341 if (map->clear_priv) 1342 map->clear_priv(map, map->priv); 1343 } 1344 1345 map->priv = priv; 1346 map->clear_priv = clear_priv; 1347 return 0; 1348 } 1349 1350 void *bpf_map__priv(struct bpf_map *map) 1351 { 1352 return map ? map->priv : ERR_PTR(-EINVAL); 1353 } 1354 1355 struct bpf_map * 1356 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) 1357 { 1358 size_t idx; 1359 struct bpf_map *s, *e; 1360 1361 if (!obj || !obj->maps) 1362 return NULL; 1363 1364 s = obj->maps; 1365 e = obj->maps + obj->nr_maps; 1366 1367 if (prev == NULL) 1368 return s; 1369 1370 if ((prev < s) || (prev >= e)) { 1371 pr_warning("error in %s: map handler doesn't belong to object\n", 1372 __func__); 1373 return NULL; 1374 } 1375 1376 idx = (prev - obj->maps) + 1; 1377 if (idx >= obj->nr_maps) 1378 return NULL; 1379 return &obj->maps[idx]; 1380 } 1381 1382 struct bpf_map * 1383 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) 1384 { 1385 struct bpf_map *pos; 1386 1387 bpf_map__for_each(pos, obj) { 1388 if (pos->name && !strcmp(pos->name, name)) 1389 return pos; 1390 } 1391 return NULL; 1392 } 1393