1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* 3 * BPF static linker 4 * 5 * Copyright (c) 2021 Facebook 6 */ 7 #include <stdbool.h> 8 #include <stddef.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <unistd.h> 13 #include <errno.h> 14 #include <linux/err.h> 15 #include <linux/btf.h> 16 #include <elf.h> 17 #include <libelf.h> 18 #include <gelf.h> 19 #include <fcntl.h> 20 #include "libbpf.h" 21 #include "btf.h" 22 #include "libbpf_internal.h" 23 #include "strset.h" 24 25 struct src_sec { 26 const char *sec_name; 27 /* positional (not necessarily ELF) index in an array of sections */ 28 int id; 29 /* positional (not necessarily ELF) index of a matching section in a final object file */ 30 int dst_id; 31 /* section data offset in a matching output section */ 32 int dst_off; 33 /* whether section is omitted from the final ELF file */ 34 bool skipped; 35 /* whether section is an ephemeral section, not mapped to an ELF section */ 36 bool ephemeral; 37 38 /* ELF info */ 39 size_t sec_idx; 40 Elf_Scn *scn; 41 Elf64_Shdr *shdr; 42 Elf_Data *data; 43 44 /* corresponding BTF DATASEC type ID */ 45 int sec_type_id; 46 }; 47 48 struct src_obj { 49 const char *filename; 50 int fd; 51 Elf *elf; 52 /* Section header strings section index */ 53 size_t shstrs_sec_idx; 54 /* SYMTAB section index */ 55 size_t symtab_sec_idx; 56 57 struct btf *btf; 58 struct btf_ext *btf_ext; 59 60 /* List of sections (including ephemeral). Slot zero is unused. */ 61 struct src_sec *secs; 62 int sec_cnt; 63 64 /* mapping of symbol indices from src to dst ELF */ 65 int *sym_map; 66 /* mapping from the src BTF type IDs to dst ones */ 67 int *btf_type_map; 68 }; 69 70 /* single .BTF.ext data section */ 71 struct btf_ext_sec_data { 72 size_t rec_cnt; 73 __u32 rec_sz; 74 void *recs; 75 }; 76 77 struct dst_sec { 78 char *sec_name; 79 /* positional (not necessarily ELF) index in an array of sections */ 80 int id; 81 82 /* ELF info */ 83 size_t sec_idx; 84 Elf_Scn *scn; 85 Elf64_Shdr *shdr; 86 Elf_Data *data; 87 88 /* final output section size */ 89 int sec_sz; 90 /* final output contents of the section */ 91 void *raw_data; 92 93 /* corresponding STT_SECTION symbol index in SYMTAB */ 94 int sec_sym_idx; 95 96 /* section's DATASEC variable info, emitted on BTF finalization */ 97 int sec_var_cnt; 98 struct btf_var_secinfo *sec_vars; 99 100 /* section's .BTF.ext data */ 101 struct btf_ext_sec_data func_info; 102 struct btf_ext_sec_data line_info; 103 struct btf_ext_sec_data core_relo_info; 104 }; 105 106 struct bpf_linker { 107 char *filename; 108 int fd; 109 Elf *elf; 110 Elf64_Ehdr *elf_hdr; 111 112 /* Output sections metadata */ 113 struct dst_sec *secs; 114 int sec_cnt; 115 116 struct strset *strtab_strs; /* STRTAB unique strings */ 117 size_t strtab_sec_idx; /* STRTAB section index */ 118 size_t symtab_sec_idx; /* SYMTAB section index */ 119 120 struct btf *btf; 121 struct btf_ext *btf_ext; 122 }; 123 124 #define pr_warn_elf(fmt, ...) \ 125 do { \ 126 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1)); \ 127 } while (0) 128 129 static int init_output_elf(struct bpf_linker *linker, const char *file); 130 131 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj); 132 static int linker_sanity_check_elf(struct src_obj *obj); 133 static int linker_sanity_check_btf(struct src_obj *obj); 134 static int linker_sanity_check_btf_ext(struct src_obj *obj); 135 static int linker_fixup_btf(struct src_obj *obj); 136 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj); 137 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj); 138 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj); 139 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj); 140 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj); 141 142 static int finalize_btf(struct bpf_linker *linker); 143 static int finalize_btf_ext(struct bpf_linker *linker); 144 145 void bpf_linker__free(struct bpf_linker *linker) 146 { 147 int i; 148 149 if (!linker) 150 return; 151 152 free(linker->filename); 153 154 if (linker->elf) 155 elf_end(linker->elf); 156 157 if (linker->fd >= 0) 158 close(linker->fd); 159 160 strset__free(linker->strtab_strs); 161 162 btf__free(linker->btf); 163 btf_ext__free(linker->btf_ext); 164 165 for (i = 1; i < linker->sec_cnt; i++) { 166 struct dst_sec *sec = &linker->secs[i]; 167 168 free(sec->sec_name); 169 free(sec->raw_data); 170 free(sec->sec_vars); 171 172 free(sec->func_info.recs); 173 free(sec->line_info.recs); 174 free(sec->core_relo_info.recs); 175 } 176 free(linker->secs); 177 178 free(linker); 179 } 180 181 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts) 182 { 183 struct bpf_linker *linker; 184 int err; 185 186 if (!OPTS_VALID(opts, bpf_linker_opts)) 187 return NULL; 188 189 if (elf_version(EV_CURRENT) == EV_NONE) { 190 pr_warn_elf("libelf initialization failed"); 191 return NULL; 192 } 193 194 linker = calloc(1, sizeof(*linker)); 195 if (!linker) 196 return NULL; 197 198 linker->fd = -1; 199 200 err = init_output_elf(linker, filename); 201 if (err) 202 goto err_out; 203 204 return linker; 205 206 err_out: 207 bpf_linker__free(linker); 208 return NULL; 209 } 210 211 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name) 212 { 213 struct dst_sec *secs = linker->secs, *sec; 214 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2; 215 216 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); 217 if (!secs) 218 return NULL; 219 220 /* zero out newly allocated memory */ 221 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs)); 222 223 linker->secs = secs; 224 linker->sec_cnt = new_cnt; 225 226 sec = &linker->secs[new_cnt - 1]; 227 sec->id = new_cnt - 1; 228 sec->sec_name = strdup(sec_name); 229 if (!sec->sec_name) 230 return NULL; 231 232 return sec; 233 } 234 235 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx) 236 { 237 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; 238 Elf64_Sym *syms, *sym; 239 size_t sym_cnt = symtab->sec_sz / sizeof(*sym); 240 241 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym)); 242 if (!syms) 243 return NULL; 244 245 sym = &syms[sym_cnt]; 246 memset(sym, 0, sizeof(*sym)); 247 248 symtab->raw_data = syms; 249 symtab->sec_sz += sizeof(*sym); 250 symtab->shdr->sh_size += sizeof(*sym); 251 symtab->data->d_size += sizeof(*sym); 252 253 if (sym_idx) 254 *sym_idx = sym_cnt; 255 256 return sym; 257 } 258 259 static int init_output_elf(struct bpf_linker *linker, const char *file) 260 { 261 int err, str_off; 262 Elf64_Sym *init_sym; 263 struct dst_sec *sec; 264 265 linker->filename = strdup(file); 266 if (!linker->filename) 267 return -ENOMEM; 268 269 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0644); 270 if (linker->fd < 0) { 271 err = -errno; 272 pr_warn("failed to create '%s': %d\n", file, err); 273 return err; 274 } 275 276 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL); 277 if (!linker->elf) { 278 pr_warn_elf("failed to create ELF object"); 279 return -EINVAL; 280 } 281 282 /* ELF header */ 283 linker->elf_hdr = elf64_newehdr(linker->elf); 284 if (!linker->elf_hdr){ 285 pr_warn_elf("failed to create ELF header"); 286 return -EINVAL; 287 } 288 289 linker->elf_hdr->e_machine = EM_BPF; 290 linker->elf_hdr->e_type = ET_REL; 291 #if __BYTE_ORDER == __LITTLE_ENDIAN 292 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB; 293 #elif __BYTE_ORDER == __BIG_ENDIAN 294 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB; 295 #else 296 #error "Unknown __BYTE_ORDER" 297 #endif 298 299 /* STRTAB */ 300 /* initialize strset with an empty string to conform to ELF */ 301 linker->strtab_strs = strset__new(INT_MAX, "", sizeof("")); 302 if (libbpf_get_error(linker->strtab_strs)) 303 return libbpf_get_error(linker->strtab_strs); 304 305 sec = add_dst_sec(linker, ".strtab"); 306 if (!sec) 307 return -ENOMEM; 308 309 sec->scn = elf_newscn(linker->elf); 310 if (!sec->scn) { 311 pr_warn_elf("failed to create STRTAB section"); 312 return -EINVAL; 313 } 314 315 sec->shdr = elf64_getshdr(sec->scn); 316 if (!sec->shdr) 317 return -EINVAL; 318 319 sec->data = elf_newdata(sec->scn); 320 if (!sec->data) { 321 pr_warn_elf("failed to create STRTAB data"); 322 return -EINVAL; 323 } 324 325 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); 326 if (str_off < 0) 327 return str_off; 328 329 sec->sec_idx = elf_ndxscn(sec->scn); 330 linker->elf_hdr->e_shstrndx = sec->sec_idx; 331 linker->strtab_sec_idx = sec->sec_idx; 332 333 sec->shdr->sh_name = str_off; 334 sec->shdr->sh_type = SHT_STRTAB; 335 sec->shdr->sh_flags = SHF_STRINGS; 336 sec->shdr->sh_offset = 0; 337 sec->shdr->sh_link = 0; 338 sec->shdr->sh_info = 0; 339 sec->shdr->sh_addralign = 1; 340 sec->shdr->sh_size = sec->sec_sz = 0; 341 sec->shdr->sh_entsize = 0; 342 343 /* SYMTAB */ 344 sec = add_dst_sec(linker, ".symtab"); 345 if (!sec) 346 return -ENOMEM; 347 348 sec->scn = elf_newscn(linker->elf); 349 if (!sec->scn) { 350 pr_warn_elf("failed to create SYMTAB section"); 351 return -EINVAL; 352 } 353 354 sec->shdr = elf64_getshdr(sec->scn); 355 if (!sec->shdr) 356 return -EINVAL; 357 358 sec->data = elf_newdata(sec->scn); 359 if (!sec->data) { 360 pr_warn_elf("failed to create SYMTAB data"); 361 return -EINVAL; 362 } 363 364 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); 365 if (str_off < 0) 366 return str_off; 367 368 sec->sec_idx = elf_ndxscn(sec->scn); 369 linker->symtab_sec_idx = sec->sec_idx; 370 371 sec->shdr->sh_name = str_off; 372 sec->shdr->sh_type = SHT_SYMTAB; 373 sec->shdr->sh_flags = 0; 374 sec->shdr->sh_offset = 0; 375 sec->shdr->sh_link = linker->strtab_sec_idx; 376 /* sh_info should be one greater than the index of the last local 377 * symbol (i.e., binding is STB_LOCAL). But why and who cares? 378 */ 379 sec->shdr->sh_info = 0; 380 sec->shdr->sh_addralign = 8; 381 sec->shdr->sh_entsize = sizeof(Elf64_Sym); 382 383 /* .BTF */ 384 linker->btf = btf__new_empty(); 385 err = libbpf_get_error(linker->btf); 386 if (err) 387 return err; 388 389 /* add the special all-zero symbol */ 390 init_sym = add_new_sym(linker, NULL); 391 if (!init_sym) 392 return -EINVAL; 393 394 init_sym->st_name = 0; 395 init_sym->st_info = 0; 396 init_sym->st_other = 0; 397 init_sym->st_shndx = SHN_UNDEF; 398 init_sym->st_value = 0; 399 init_sym->st_size = 0; 400 401 return 0; 402 } 403 404 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename) 405 { 406 struct src_obj obj = {}; 407 int err = 0; 408 409 if (!linker->elf) 410 return -EINVAL; 411 412 err = err ?: linker_load_obj_file(linker, filename, &obj); 413 err = err ?: linker_append_sec_data(linker, &obj); 414 err = err ?: linker_append_elf_syms(linker, &obj); 415 err = err ?: linker_append_elf_relos(linker, &obj); 416 err = err ?: linker_append_btf(linker, &obj); 417 err = err ?: linker_append_btf_ext(linker, &obj); 418 419 /* free up src_obj resources */ 420 free(obj.btf_type_map); 421 btf__free(obj.btf); 422 btf_ext__free(obj.btf_ext); 423 free(obj.secs); 424 free(obj.sym_map); 425 if (obj.elf) 426 elf_end(obj.elf); 427 if (obj.fd >= 0) 428 close(obj.fd); 429 430 return err; 431 } 432 433 static bool is_dwarf_sec_name(const char *name) 434 { 435 /* approximation, but the actual list is too long */ 436 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; 437 } 438 439 static bool is_ignored_sec(struct src_sec *sec) 440 { 441 Elf64_Shdr *shdr = sec->shdr; 442 const char *name = sec->sec_name; 443 444 /* no special handling of .strtab */ 445 if (shdr->sh_type == SHT_STRTAB) 446 return true; 447 448 /* ignore .llvm_addrsig section as well */ 449 if (shdr->sh_type == SHT_LLVM_ADDRSIG) 450 return true; 451 452 /* no subprograms will lead to an empty .text section, ignore it */ 453 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 && 454 strcmp(sec->sec_name, ".text") == 0) 455 return true; 456 457 /* DWARF sections */ 458 if (is_dwarf_sec_name(sec->sec_name)) 459 return true; 460 461 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { 462 name += sizeof(".rel") - 1; 463 /* DWARF section relocations */ 464 if (is_dwarf_sec_name(name)) 465 return true; 466 467 /* .BTF and .BTF.ext don't need relocations */ 468 if (strcmp(name, BTF_ELF_SEC) == 0 || 469 strcmp(name, BTF_EXT_ELF_SEC) == 0) 470 return true; 471 } 472 473 return false; 474 } 475 476 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name) 477 { 478 struct src_sec *secs = obj->secs, *sec; 479 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2; 480 481 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); 482 if (!secs) 483 return NULL; 484 485 /* zero out newly allocated memory */ 486 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs)); 487 488 obj->secs = secs; 489 obj->sec_cnt = new_cnt; 490 491 sec = &obj->secs[new_cnt - 1]; 492 sec->id = new_cnt - 1; 493 sec->sec_name = sec_name; 494 495 return sec; 496 } 497 498 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj) 499 { 500 #if __BYTE_ORDER == __LITTLE_ENDIAN 501 const int host_endianness = ELFDATA2LSB; 502 #elif __BYTE_ORDER == __BIG_ENDIAN 503 const int host_endianness = ELFDATA2MSB; 504 #else 505 #error "Unknown __BYTE_ORDER" 506 #endif 507 int err = 0; 508 Elf_Scn *scn; 509 Elf_Data *data; 510 Elf64_Ehdr *ehdr; 511 Elf64_Shdr *shdr; 512 struct src_sec *sec; 513 514 pr_debug("linker: adding object file '%s'...\n", filename); 515 516 obj->filename = filename; 517 518 obj->fd = open(filename, O_RDONLY); 519 if (obj->fd < 0) { 520 err = -errno; 521 pr_warn("failed to open file '%s': %d\n", filename, err); 522 return err; 523 } 524 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL); 525 if (!obj->elf) { 526 err = -errno; 527 pr_warn_elf("failed to parse ELF file '%s'", filename); 528 return err; 529 } 530 531 /* Sanity check ELF file high-level properties */ 532 ehdr = elf64_getehdr(obj->elf); 533 if (!ehdr) { 534 err = -errno; 535 pr_warn_elf("failed to get ELF header for %s", filename); 536 return err; 537 } 538 if (ehdr->e_ident[EI_DATA] != host_endianness) { 539 err = -EOPNOTSUPP; 540 pr_warn_elf("unsupported byte order of ELF file %s", filename); 541 return err; 542 } 543 if (ehdr->e_type != ET_REL 544 || ehdr->e_machine != EM_BPF 545 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) { 546 err = -EOPNOTSUPP; 547 pr_warn_elf("unsupported kind of ELF file %s", filename); 548 return err; 549 } 550 551 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) { 552 err = -errno; 553 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename); 554 return err; 555 } 556 557 scn = NULL; 558 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) { 559 size_t sec_idx = elf_ndxscn(scn); 560 const char *sec_name; 561 562 shdr = elf64_getshdr(scn); 563 if (!shdr) { 564 err = -errno; 565 pr_warn_elf("failed to get section #%zu header for %s", 566 sec_idx, filename); 567 return err; 568 } 569 570 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name); 571 if (!sec_name) { 572 err = -errno; 573 pr_warn_elf("failed to get section #%zu name for %s", 574 sec_idx, filename); 575 return err; 576 } 577 578 data = elf_getdata(scn, 0); 579 if (!data) { 580 err = -errno; 581 pr_warn_elf("failed to get section #%zu (%s) data from %s", 582 sec_idx, sec_name, filename); 583 return err; 584 } 585 586 sec = add_src_sec(obj, sec_name); 587 if (!sec) 588 return -ENOMEM; 589 590 sec->scn = scn; 591 sec->shdr = shdr; 592 sec->data = data; 593 sec->sec_idx = elf_ndxscn(scn); 594 595 if (is_ignored_sec(sec)) { 596 sec->skipped = true; 597 continue; 598 } 599 600 switch (shdr->sh_type) { 601 case SHT_SYMTAB: 602 if (obj->symtab_sec_idx) { 603 err = -EOPNOTSUPP; 604 pr_warn("multiple SYMTAB sections found, not supported\n"); 605 return err; 606 } 607 obj->symtab_sec_idx = sec_idx; 608 break; 609 case SHT_STRTAB: 610 /* we'll construct our own string table */ 611 break; 612 case SHT_PROGBITS: 613 if (strcmp(sec_name, BTF_ELF_SEC) == 0) { 614 obj->btf = btf__new(data->d_buf, shdr->sh_size); 615 err = libbpf_get_error(obj->btf); 616 if (err) { 617 pr_warn("failed to parse .BTF from %s: %d\n", filename, err); 618 return err; 619 } 620 sec->skipped = true; 621 continue; 622 } 623 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) { 624 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size); 625 err = libbpf_get_error(obj->btf_ext); 626 if (err) { 627 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err); 628 return err; 629 } 630 sec->skipped = true; 631 continue; 632 } 633 634 /* data & code */ 635 break; 636 case SHT_NOBITS: 637 /* BSS */ 638 break; 639 case SHT_REL: 640 /* relocations */ 641 break; 642 default: 643 pr_warn("unrecognized section #%zu (%s) in %s\n", 644 sec_idx, sec_name, filename); 645 err = -EINVAL; 646 return err; 647 } 648 } 649 650 err = err ?: linker_sanity_check_elf(obj); 651 err = err ?: linker_sanity_check_btf(obj); 652 err = err ?: linker_sanity_check_btf_ext(obj); 653 err = err ?: linker_fixup_btf(obj); 654 655 return err; 656 } 657 658 static bool is_pow_of_2(size_t x) 659 { 660 return x && (x & (x - 1)) == 0; 661 } 662 663 static int linker_sanity_check_elf(struct src_obj *obj) 664 { 665 struct src_sec *sec, *link_sec; 666 int i, j, n; 667 668 if (!obj->symtab_sec_idx) { 669 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename); 670 return -EINVAL; 671 } 672 if (!obj->shstrs_sec_idx) { 673 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename); 674 return -EINVAL; 675 } 676 677 for (i = 1; i < obj->sec_cnt; i++) { 678 sec = &obj->secs[i]; 679 680 if (sec->sec_name[0] == '\0') { 681 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename); 682 return -EINVAL; 683 } 684 685 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) 686 return -EINVAL; 687 if (sec->shdr->sh_addralign != sec->data->d_align) 688 return -EINVAL; 689 690 if (sec->shdr->sh_size != sec->data->d_size) 691 return -EINVAL; 692 693 switch (sec->shdr->sh_type) { 694 case SHT_SYMTAB: { 695 Elf64_Sym *sym; 696 697 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym)) 698 return -EINVAL; 699 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) 700 return -EINVAL; 701 702 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) { 703 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n", 704 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 705 return -EINVAL; 706 } 707 link_sec = &obj->secs[sec->shdr->sh_link]; 708 if (link_sec->shdr->sh_type != SHT_STRTAB) { 709 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n", 710 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 711 return -EINVAL; 712 } 713 714 n = sec->shdr->sh_size / sec->shdr->sh_entsize; 715 sym = sec->data->d_buf; 716 for (j = 0; j < n; j++, sym++) { 717 if (sym->st_shndx 718 && sym->st_shndx < SHN_LORESERVE 719 && sym->st_shndx >= obj->sec_cnt) { 720 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n", 721 j, sec->sec_idx, (size_t)sym->st_shndx, obj->filename); 722 return -EINVAL; 723 } 724 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION) { 725 if (sym->st_value != 0) 726 return -EINVAL; 727 } 728 } 729 break; 730 } 731 case SHT_STRTAB: 732 break; 733 case SHT_PROGBITS: 734 if (sec->shdr->sh_flags & SHF_EXECINSTR) { 735 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) 736 return -EINVAL; 737 } 738 break; 739 case SHT_NOBITS: 740 break; 741 case SHT_REL: { 742 Elf64_Rel *relo; 743 struct src_sec *sym_sec; 744 745 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel)) 746 return -EINVAL; 747 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) 748 return -EINVAL; 749 750 /* SHT_REL's sh_link should point to SYMTAB */ 751 if (sec->shdr->sh_link != obj->symtab_sec_idx) { 752 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n", 753 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 754 return -EINVAL; 755 } 756 757 /* SHT_REL's sh_info points to relocated section */ 758 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) { 759 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n", 760 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); 761 return -EINVAL; 762 } 763 link_sec = &obj->secs[sec->shdr->sh_info]; 764 765 /* .rel<secname> -> <secname> pattern is followed */ 766 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0 767 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) { 768 pr_warn("ELF relo section #%zu name has invalid name in %s\n", 769 sec->sec_idx, obj->filename); 770 return -EINVAL; 771 } 772 773 /* don't further validate relocations for ignored sections */ 774 if (link_sec->skipped) 775 break; 776 777 /* relocatable section is data or instructions */ 778 if (link_sec->shdr->sh_type != SHT_PROGBITS 779 && link_sec->shdr->sh_type != SHT_NOBITS) { 780 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n", 781 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); 782 return -EINVAL; 783 } 784 785 /* check sanity of each relocation */ 786 n = sec->shdr->sh_size / sec->shdr->sh_entsize; 787 relo = sec->data->d_buf; 788 sym_sec = &obj->secs[obj->symtab_sec_idx]; 789 for (j = 0; j < n; j++, relo++) { 790 size_t sym_idx = ELF64_R_SYM(relo->r_info); 791 size_t sym_type = ELF64_R_TYPE(relo->r_info); 792 793 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32) { 794 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n", 795 j, sec->sec_idx, sym_type, obj->filename); 796 return -EINVAL; 797 } 798 799 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) { 800 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n", 801 j, sec->sec_idx, sym_idx, obj->filename); 802 return -EINVAL; 803 } 804 805 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) { 806 if (relo->r_offset % sizeof(struct bpf_insn) != 0) { 807 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n", 808 j, sec->sec_idx, sym_idx, obj->filename); 809 return -EINVAL; 810 } 811 } 812 } 813 break; 814 } 815 case SHT_LLVM_ADDRSIG: 816 break; 817 default: 818 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n", 819 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename); 820 return -EINVAL; 821 } 822 } 823 824 return 0; 825 } 826 827 static int check_btf_type_id(__u32 *type_id, void *ctx) 828 { 829 struct btf *btf = ctx; 830 831 if (*type_id > btf__get_nr_types(btf)) 832 return -EINVAL; 833 834 return 0; 835 } 836 837 static int check_btf_str_off(__u32 *str_off, void *ctx) 838 { 839 struct btf *btf = ctx; 840 const char *s; 841 842 s = btf__str_by_offset(btf, *str_off); 843 844 if (!s) 845 return -EINVAL; 846 847 return 0; 848 } 849 850 static int linker_sanity_check_btf(struct src_obj *obj) 851 { 852 struct btf_type *t; 853 int i, n, err = 0; 854 855 if (!obj->btf) 856 return 0; 857 858 n = btf__get_nr_types(obj->btf); 859 for (i = 1; i <= n; i++) { 860 t = btf_type_by_id(obj->btf, i); 861 862 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf); 863 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf); 864 if (err) 865 return err; 866 } 867 868 return 0; 869 } 870 871 static int linker_sanity_check_btf_ext(struct src_obj *obj) 872 { 873 int err = 0; 874 875 if (!obj->btf_ext) 876 return 0; 877 878 /* can't use .BTF.ext without .BTF */ 879 if (!obj->btf) 880 return -EINVAL; 881 882 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf); 883 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf); 884 if (err) 885 return err; 886 887 return 0; 888 } 889 890 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec) 891 { 892 Elf_Scn *scn; 893 Elf_Data *data; 894 Elf64_Shdr *shdr; 895 int name_off; 896 897 dst_sec->sec_sz = 0; 898 dst_sec->sec_idx = 0; 899 900 /* ephemeral sections are just thin section shells lacking most parts */ 901 if (src_sec->ephemeral) 902 return 0; 903 904 scn = elf_newscn(linker->elf); 905 if (!scn) 906 return -1; 907 data = elf_newdata(scn); 908 if (!data) 909 return -1; 910 shdr = elf64_getshdr(scn); 911 if (!shdr) 912 return -1; 913 914 dst_sec->scn = scn; 915 dst_sec->shdr = shdr; 916 dst_sec->data = data; 917 dst_sec->sec_idx = elf_ndxscn(scn); 918 919 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name); 920 if (name_off < 0) 921 return name_off; 922 923 shdr->sh_name = name_off; 924 shdr->sh_type = src_sec->shdr->sh_type; 925 shdr->sh_flags = src_sec->shdr->sh_flags; 926 shdr->sh_size = 0; 927 /* sh_link and sh_info have different meaning for different types of 928 * sections, so we leave it up to the caller code to fill them in, if 929 * necessary 930 */ 931 shdr->sh_link = 0; 932 shdr->sh_info = 0; 933 shdr->sh_addralign = src_sec->shdr->sh_addralign; 934 shdr->sh_entsize = src_sec->shdr->sh_entsize; 935 936 data->d_type = src_sec->data->d_type; 937 data->d_size = 0; 938 data->d_buf = NULL; 939 data->d_align = src_sec->data->d_align; 940 data->d_off = 0; 941 942 return 0; 943 } 944 945 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name) 946 { 947 struct dst_sec *sec; 948 int i; 949 950 for (i = 1; i < linker->sec_cnt; i++) { 951 sec = &linker->secs[i]; 952 953 if (strcmp(sec->sec_name, sec_name) == 0) 954 return sec; 955 } 956 957 return NULL; 958 } 959 960 static bool secs_match(struct dst_sec *dst, struct src_sec *src) 961 { 962 if (dst->shdr->sh_type != src->shdr->sh_type) { 963 pr_warn("sec %s types mismatch\n", dst->sec_name); 964 return false; 965 } 966 if (dst->shdr->sh_flags != src->shdr->sh_flags) { 967 pr_warn("sec %s flags mismatch\n", dst->sec_name); 968 return false; 969 } 970 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) { 971 pr_warn("sec %s entsize mismatch\n", dst->sec_name); 972 return false; 973 } 974 975 return true; 976 } 977 978 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec) 979 { 980 if (dst_sec->sec_sz != src_sec->shdr->sh_size) 981 return false; 982 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0) 983 return false; 984 return true; 985 } 986 987 static int extend_sec(struct dst_sec *dst, struct src_sec *src) 988 { 989 void *tmp; 990 size_t dst_align = dst->shdr->sh_addralign; 991 size_t src_align = src->shdr->sh_addralign; 992 size_t dst_align_sz, dst_final_sz; 993 994 if (dst_align == 0) 995 dst_align = 1; 996 if (dst_align < src_align) 997 dst_align = src_align; 998 999 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align; 1000 1001 /* no need to re-align final size */ 1002 dst_final_sz = dst_align_sz + src->shdr->sh_size; 1003 1004 if (src->shdr->sh_type != SHT_NOBITS) { 1005 tmp = realloc(dst->raw_data, dst_final_sz); 1006 if (!tmp) 1007 return -ENOMEM; 1008 dst->raw_data = tmp; 1009 1010 /* pad dst section, if it's alignment forced size increase */ 1011 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz); 1012 /* now copy src data at a properly aligned offset */ 1013 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size); 1014 } 1015 1016 dst->sec_sz = dst_final_sz; 1017 dst->shdr->sh_size = dst_final_sz; 1018 dst->data->d_size = dst_final_sz; 1019 1020 dst->shdr->sh_addralign = dst_align; 1021 dst->data->d_align = dst_align; 1022 1023 src->dst_off = dst_align_sz; 1024 1025 return 0; 1026 } 1027 1028 static bool is_data_sec(struct src_sec *sec) 1029 { 1030 if (!sec || sec->skipped) 1031 return false; 1032 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */ 1033 if (sec->ephemeral) 1034 return true; 1035 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS; 1036 } 1037 1038 static bool is_relo_sec(struct src_sec *sec) 1039 { 1040 if (!sec || sec->skipped || sec->ephemeral) 1041 return false; 1042 return sec->shdr->sh_type == SHT_REL; 1043 } 1044 1045 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj) 1046 { 1047 int i, err; 1048 1049 for (i = 1; i < obj->sec_cnt; i++) { 1050 struct src_sec *src_sec; 1051 struct dst_sec *dst_sec; 1052 1053 src_sec = &obj->secs[i]; 1054 if (!is_data_sec(src_sec)) 1055 continue; 1056 1057 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); 1058 if (!dst_sec) { 1059 dst_sec = add_dst_sec(linker, src_sec->sec_name); 1060 if (!dst_sec) 1061 return -ENOMEM; 1062 err = init_sec(linker, dst_sec, src_sec); 1063 if (err) { 1064 pr_warn("failed to init section '%s'\n", src_sec->sec_name); 1065 return err; 1066 } 1067 } else { 1068 if (!secs_match(dst_sec, src_sec)) { 1069 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name); 1070 return -1; 1071 } 1072 1073 /* "license" and "version" sections are deduped */ 1074 if (strcmp(src_sec->sec_name, "license") == 0 1075 || strcmp(src_sec->sec_name, "version") == 0) { 1076 if (!sec_content_is_same(dst_sec, src_sec)) { 1077 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name); 1078 return -EINVAL; 1079 } 1080 src_sec->skipped = true; 1081 src_sec->dst_id = dst_sec->id; 1082 continue; 1083 } 1084 } 1085 1086 /* record mapped section index */ 1087 src_sec->dst_id = dst_sec->id; 1088 1089 if (src_sec->ephemeral) 1090 continue; 1091 1092 err = extend_sec(dst_sec, src_sec); 1093 if (err) 1094 return err; 1095 } 1096 1097 return 0; 1098 } 1099 1100 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj) 1101 { 1102 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; 1103 Elf64_Sym *sym = symtab->data->d_buf, *dst_sym; 1104 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; 1105 int str_sec_idx = symtab->shdr->sh_link; 1106 1107 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map)); 1108 if (!obj->sym_map) 1109 return -ENOMEM; 1110 1111 for (i = 0; i < n; i++, sym++) { 1112 struct src_sec *src_sec = NULL; 1113 struct dst_sec *dst_sec = NULL; 1114 const char *sym_name; 1115 size_t dst_sym_idx; 1116 int name_off; 1117 1118 /* we already have all-zero initial symbol */ 1119 if (sym->st_name == 0 && sym->st_info == 0 && 1120 sym->st_other == 0 && sym->st_shndx == SHN_UNDEF && 1121 sym->st_value == 0 && sym->st_size ==0) 1122 continue; 1123 1124 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); 1125 if (!sym_name) { 1126 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename); 1127 return -1; 1128 } 1129 1130 if (sym->st_shndx && sym->st_shndx < SHN_LORESERVE) { 1131 src_sec = &obj->secs[sym->st_shndx]; 1132 if (src_sec->skipped) 1133 continue; 1134 dst_sec = &linker->secs[src_sec->dst_id]; 1135 1136 /* allow only one STT_SECTION symbol per section */ 1137 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sec->sec_sym_idx) { 1138 obj->sym_map[i] = dst_sec->sec_sym_idx; 1139 continue; 1140 } 1141 } 1142 1143 name_off = strset__add_str(linker->strtab_strs, sym_name); 1144 if (name_off < 0) 1145 return name_off; 1146 1147 dst_sym = add_new_sym(linker, &dst_sym_idx); 1148 if (!dst_sym) 1149 return -ENOMEM; 1150 1151 dst_sym->st_name = name_off; 1152 dst_sym->st_info = sym->st_info; 1153 dst_sym->st_other = sym->st_other; 1154 dst_sym->st_shndx = src_sec ? dst_sec->sec_idx : sym->st_shndx; 1155 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; 1156 dst_sym->st_size = sym->st_size; 1157 1158 obj->sym_map[i] = dst_sym_idx; 1159 1160 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sym) { 1161 dst_sec->sec_sym_idx = dst_sym_idx; 1162 dst_sym->st_value = 0; 1163 } 1164 1165 } 1166 1167 return 0; 1168 } 1169 1170 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj) 1171 { 1172 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx]; 1173 struct dst_sec *dst_symtab = &linker->secs[linker->symtab_sec_idx]; 1174 int i, err; 1175 1176 for (i = 1; i < obj->sec_cnt; i++) { 1177 struct src_sec *src_sec, *src_linked_sec; 1178 struct dst_sec *dst_sec, *dst_linked_sec; 1179 Elf64_Rel *src_rel, *dst_rel; 1180 int j, n; 1181 1182 src_sec = &obj->secs[i]; 1183 if (!is_relo_sec(src_sec)) 1184 continue; 1185 1186 /* shdr->sh_info points to relocatable section */ 1187 src_linked_sec = &obj->secs[src_sec->shdr->sh_info]; 1188 if (src_linked_sec->skipped) 1189 continue; 1190 1191 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); 1192 if (!dst_sec) { 1193 dst_sec = add_dst_sec(linker, src_sec->sec_name); 1194 if (!dst_sec) 1195 return -ENOMEM; 1196 err = init_sec(linker, dst_sec, src_sec); 1197 if (err) { 1198 pr_warn("failed to init section '%s'\n", src_sec->sec_name); 1199 return err; 1200 } 1201 } else if (!secs_match(dst_sec, src_sec)) { 1202 pr_warn("Secs %s are not compatible\n", src_sec->sec_name); 1203 return -1; 1204 } 1205 1206 /* shdr->sh_link points to SYMTAB */ 1207 dst_sec->shdr->sh_link = linker->symtab_sec_idx; 1208 1209 /* shdr->sh_info points to relocated section */ 1210 dst_linked_sec = &linker->secs[src_linked_sec->dst_id]; 1211 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx; 1212 1213 src_sec->dst_id = dst_sec->id; 1214 err = extend_sec(dst_sec, src_sec); 1215 if (err) 1216 return err; 1217 1218 src_rel = src_sec->data->d_buf; 1219 dst_rel = dst_sec->raw_data + src_sec->dst_off; 1220 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize; 1221 for (j = 0; j < n; j++, src_rel++, dst_rel++) { 1222 size_t src_sym_idx = ELF64_R_SYM(src_rel->r_info); 1223 size_t sym_type = ELF64_R_TYPE(src_rel->r_info); 1224 Elf64_Sym *src_sym, *dst_sym; 1225 size_t dst_sym_idx; 1226 1227 src_sym_idx = ELF64_R_SYM(src_rel->r_info); 1228 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx; 1229 1230 dst_sym_idx = obj->sym_map[src_sym_idx]; 1231 dst_sym = dst_symtab->raw_data + sizeof(*dst_sym) * dst_sym_idx; 1232 dst_rel->r_offset += src_linked_sec->dst_off; 1233 sym_type = ELF64_R_TYPE(src_rel->r_info); 1234 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type); 1235 1236 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) { 1237 struct src_sec *sec = &obj->secs[src_sym->st_shndx]; 1238 struct bpf_insn *insn; 1239 1240 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) { 1241 /* calls to the very first static function inside 1242 * .text section at offset 0 will 1243 * reference section symbol, not the 1244 * function symbol. Fix that up, 1245 * otherwise it won't be possible to 1246 * relocate calls to two different 1247 * static functions with the same name 1248 * (rom two different object files) 1249 */ 1250 insn = dst_linked_sec->raw_data + dst_rel->r_offset; 1251 if (insn->code == (BPF_JMP | BPF_CALL)) 1252 insn->imm += sec->dst_off / sizeof(struct bpf_insn); 1253 else 1254 insn->imm += sec->dst_off; 1255 } else { 1256 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n"); 1257 return -EINVAL; 1258 } 1259 } 1260 1261 } 1262 } 1263 1264 return 0; 1265 } 1266 1267 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name) 1268 { 1269 struct src_sec *sec; 1270 int i; 1271 1272 for (i = 1; i < obj->sec_cnt; i++) { 1273 sec = &obj->secs[i]; 1274 1275 if (strcmp(sec->sec_name, sec_name) == 0) 1276 return sec; 1277 } 1278 1279 return NULL; 1280 } 1281 1282 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx, 1283 int sym_type, const char *sym_name) 1284 { 1285 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; 1286 Elf64_Sym *sym = symtab->data->d_buf; 1287 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; 1288 int str_sec_idx = symtab->shdr->sh_link; 1289 const char *name; 1290 1291 for (i = 0; i < n; i++, sym++) { 1292 if (sym->st_shndx != sec_idx) 1293 continue; 1294 if (ELF64_ST_TYPE(sym->st_info) != sym_type) 1295 continue; 1296 1297 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); 1298 if (!name) 1299 return NULL; 1300 1301 if (strcmp(sym_name, name) != 0) 1302 continue; 1303 1304 return sym; 1305 } 1306 1307 return NULL; 1308 } 1309 1310 static int linker_fixup_btf(struct src_obj *obj) 1311 { 1312 const char *sec_name; 1313 struct src_sec *sec; 1314 int i, j, n, m; 1315 1316 n = btf__get_nr_types(obj->btf); 1317 for (i = 1; i <= n; i++) { 1318 struct btf_var_secinfo *vi; 1319 struct btf_type *t; 1320 1321 t = btf_type_by_id(obj->btf, i); 1322 if (btf_kind(t) != BTF_KIND_DATASEC) 1323 continue; 1324 1325 sec_name = btf__str_by_offset(obj->btf, t->name_off); 1326 sec = find_src_sec_by_name(obj, sec_name); 1327 if (sec) { 1328 /* record actual section size, unless ephemeral */ 1329 if (sec->shdr) 1330 t->size = sec->shdr->sh_size; 1331 } else { 1332 /* BTF can have some sections that are not represented 1333 * in ELF, e.g., .kconfig and .ksyms, which are used 1334 * for special extern variables. Here we'll 1335 * pre-create "section shells" for them to be able to 1336 * keep track of extra per-section metadata later 1337 * (e.g., BTF variables). 1338 */ 1339 sec = add_src_sec(obj, sec_name); 1340 if (!sec) 1341 return -ENOMEM; 1342 1343 sec->ephemeral = true; 1344 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */ 1345 } 1346 1347 /* remember ELF section and its BTF type ID match */ 1348 sec->sec_type_id = i; 1349 1350 /* fix up variable offsets */ 1351 vi = btf_var_secinfos(t); 1352 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) { 1353 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type); 1354 const char *var_name = btf__str_by_offset(obj->btf, vt->name_off); 1355 int var_linkage = btf_var(vt)->linkage; 1356 Elf64_Sym *sym; 1357 1358 /* no need to patch up static or extern vars */ 1359 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED) 1360 continue; 1361 1362 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name); 1363 if (!sym) { 1364 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name); 1365 return -ENOENT; 1366 } 1367 1368 vi->offset = sym->st_value; 1369 } 1370 } 1371 1372 return 0; 1373 } 1374 1375 static int remap_type_id(__u32 *type_id, void *ctx) 1376 { 1377 int *id_map = ctx; 1378 1379 *type_id = id_map[*type_id]; 1380 1381 return 0; 1382 } 1383 1384 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) 1385 { 1386 const struct btf_type *t; 1387 int i, j, n, start_id, id; 1388 1389 if (!obj->btf) 1390 return 0; 1391 1392 start_id = btf__get_nr_types(linker->btf) + 1; 1393 n = btf__get_nr_types(obj->btf); 1394 1395 obj->btf_type_map = calloc(n + 1, sizeof(int)); 1396 if (!obj->btf_type_map) 1397 return -ENOMEM; 1398 1399 for (i = 1; i <= n; i++) { 1400 t = btf__type_by_id(obj->btf, i); 1401 1402 /* DATASECs are handled specially below */ 1403 if (btf_kind(t) == BTF_KIND_DATASEC) 1404 continue; 1405 1406 id = btf__add_type(linker->btf, obj->btf, t); 1407 if (id < 0) { 1408 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename); 1409 return id; 1410 } 1411 1412 obj->btf_type_map[i] = id; 1413 } 1414 1415 /* remap all the types except DATASECs */ 1416 n = btf__get_nr_types(linker->btf); 1417 for (i = start_id; i <= n; i++) { 1418 struct btf_type *dst_t = btf_type_by_id(linker->btf, i); 1419 1420 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map)) 1421 return -EINVAL; 1422 } 1423 1424 /* append DATASEC info */ 1425 for (i = 1; i < obj->sec_cnt; i++) { 1426 struct src_sec *src_sec; 1427 struct dst_sec *dst_sec; 1428 const struct btf_var_secinfo *src_var; 1429 struct btf_var_secinfo *dst_var; 1430 1431 src_sec = &obj->secs[i]; 1432 if (!src_sec->sec_type_id || src_sec->skipped) 1433 continue; 1434 dst_sec = &linker->secs[src_sec->dst_id]; 1435 1436 t = btf__type_by_id(obj->btf, src_sec->sec_type_id); 1437 src_var = btf_var_secinfos(t); 1438 n = btf_vlen(t); 1439 for (j = 0; j < n; j++, src_var++) { 1440 void *sec_vars = dst_sec->sec_vars; 1441 1442 sec_vars = libbpf_reallocarray(sec_vars, 1443 dst_sec->sec_var_cnt + 1, 1444 sizeof(*dst_sec->sec_vars)); 1445 if (!sec_vars) 1446 return -ENOMEM; 1447 1448 dst_sec->sec_vars = sec_vars; 1449 dst_sec->sec_var_cnt++; 1450 1451 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1]; 1452 dst_var->type = obj->btf_type_map[src_var->type]; 1453 dst_var->size = src_var->size; 1454 dst_var->offset = src_sec->dst_off + src_var->offset; 1455 } 1456 } 1457 1458 return 0; 1459 } 1460 1461 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec) 1462 { 1463 void *tmp; 1464 1465 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz); 1466 if (!tmp) 1467 return NULL; 1468 ext_data->recs = tmp; 1469 1470 tmp += ext_data->rec_cnt * ext_data->rec_sz; 1471 memcpy(tmp, src_rec, ext_data->rec_sz); 1472 1473 ext_data->rec_cnt++; 1474 1475 return tmp; 1476 } 1477 1478 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj) 1479 { 1480 const struct btf_ext_info_sec *ext_sec; 1481 const char *sec_name, *s; 1482 struct src_sec *src_sec; 1483 struct dst_sec *dst_sec; 1484 int rec_sz, str_off, i; 1485 1486 if (!obj->btf_ext) 1487 return 0; 1488 1489 rec_sz = obj->btf_ext->func_info.rec_size; 1490 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) { 1491 struct bpf_func_info_min *src_rec, *dst_rec; 1492 1493 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 1494 src_sec = find_src_sec_by_name(obj, sec_name); 1495 if (!src_sec) { 1496 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 1497 return -EINVAL; 1498 } 1499 dst_sec = &linker->secs[src_sec->dst_id]; 1500 1501 if (dst_sec->func_info.rec_sz == 0) 1502 dst_sec->func_info.rec_sz = rec_sz; 1503 if (dst_sec->func_info.rec_sz != rec_sz) { 1504 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 1505 return -EINVAL; 1506 } 1507 1508 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) { 1509 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec); 1510 if (!dst_rec) 1511 return -ENOMEM; 1512 1513 dst_rec->insn_off += src_sec->dst_off; 1514 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; 1515 } 1516 } 1517 1518 rec_sz = obj->btf_ext->line_info.rec_size; 1519 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) { 1520 struct bpf_line_info_min *src_rec, *dst_rec; 1521 1522 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 1523 src_sec = find_src_sec_by_name(obj, sec_name); 1524 if (!src_sec) { 1525 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 1526 return -EINVAL; 1527 } 1528 dst_sec = &linker->secs[src_sec->dst_id]; 1529 1530 if (dst_sec->line_info.rec_sz == 0) 1531 dst_sec->line_info.rec_sz = rec_sz; 1532 if (dst_sec->line_info.rec_sz != rec_sz) { 1533 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 1534 return -EINVAL; 1535 } 1536 1537 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) { 1538 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec); 1539 if (!dst_rec) 1540 return -ENOMEM; 1541 1542 dst_rec->insn_off += src_sec->dst_off; 1543 1544 s = btf__str_by_offset(obj->btf, src_rec->file_name_off); 1545 str_off = btf__add_str(linker->btf, s); 1546 if (str_off < 0) 1547 return -ENOMEM; 1548 dst_rec->file_name_off = str_off; 1549 1550 s = btf__str_by_offset(obj->btf, src_rec->line_off); 1551 str_off = btf__add_str(linker->btf, s); 1552 if (str_off < 0) 1553 return -ENOMEM; 1554 dst_rec->line_off = str_off; 1555 1556 /* dst_rec->line_col is fine */ 1557 } 1558 } 1559 1560 rec_sz = obj->btf_ext->core_relo_info.rec_size; 1561 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) { 1562 struct bpf_core_relo *src_rec, *dst_rec; 1563 1564 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 1565 src_sec = find_src_sec_by_name(obj, sec_name); 1566 if (!src_sec) { 1567 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 1568 return -EINVAL; 1569 } 1570 dst_sec = &linker->secs[src_sec->dst_id]; 1571 1572 if (dst_sec->core_relo_info.rec_sz == 0) 1573 dst_sec->core_relo_info.rec_sz = rec_sz; 1574 if (dst_sec->core_relo_info.rec_sz != rec_sz) { 1575 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 1576 return -EINVAL; 1577 } 1578 1579 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) { 1580 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec); 1581 if (!dst_rec) 1582 return -ENOMEM; 1583 1584 dst_rec->insn_off += src_sec->dst_off; 1585 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; 1586 1587 s = btf__str_by_offset(obj->btf, src_rec->access_str_off); 1588 str_off = btf__add_str(linker->btf, s); 1589 if (str_off < 0) 1590 return -ENOMEM; 1591 dst_rec->access_str_off = str_off; 1592 1593 /* dst_rec->kind is fine */ 1594 } 1595 } 1596 1597 return 0; 1598 } 1599 1600 int bpf_linker__finalize(struct bpf_linker *linker) 1601 { 1602 struct dst_sec *sec; 1603 size_t strs_sz; 1604 const void *strs; 1605 int err, i; 1606 1607 if (!linker->elf) 1608 return -EINVAL; 1609 1610 err = finalize_btf(linker); 1611 if (err) 1612 return err; 1613 1614 /* Finalize strings */ 1615 strs_sz = strset__data_size(linker->strtab_strs); 1616 strs = strset__data(linker->strtab_strs); 1617 1618 sec = &linker->secs[linker->strtab_sec_idx]; 1619 sec->data->d_align = 1; 1620 sec->data->d_off = 0LL; 1621 sec->data->d_buf = (void *)strs; 1622 sec->data->d_type = ELF_T_BYTE; 1623 sec->data->d_size = strs_sz; 1624 sec->shdr->sh_size = strs_sz; 1625 1626 for (i = 1; i < linker->sec_cnt; i++) { 1627 sec = &linker->secs[i]; 1628 1629 /* STRTAB is handled specially above */ 1630 if (sec->sec_idx == linker->strtab_sec_idx) 1631 continue; 1632 1633 /* special ephemeral sections (.ksyms, .kconfig, etc) */ 1634 if (!sec->scn) 1635 continue; 1636 1637 sec->data->d_buf = sec->raw_data; 1638 } 1639 1640 /* Finalize ELF layout */ 1641 if (elf_update(linker->elf, ELF_C_NULL) < 0) { 1642 err = -errno; 1643 pr_warn_elf("failed to finalize ELF layout"); 1644 return err; 1645 } 1646 1647 /* Write out final ELF contents */ 1648 if (elf_update(linker->elf, ELF_C_WRITE) < 0) { 1649 err = -errno; 1650 pr_warn_elf("failed to write ELF contents"); 1651 return err; 1652 } 1653 1654 elf_end(linker->elf); 1655 close(linker->fd); 1656 1657 linker->elf = NULL; 1658 linker->fd = -1; 1659 1660 return 0; 1661 } 1662 1663 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name, 1664 size_t align, const void *raw_data, size_t raw_sz) 1665 { 1666 Elf_Scn *scn; 1667 Elf_Data *data; 1668 Elf64_Shdr *shdr; 1669 int name_off; 1670 1671 name_off = strset__add_str(linker->strtab_strs, sec_name); 1672 if (name_off < 0) 1673 return name_off; 1674 1675 scn = elf_newscn(linker->elf); 1676 if (!scn) 1677 return -ENOMEM; 1678 data = elf_newdata(scn); 1679 if (!data) 1680 return -ENOMEM; 1681 shdr = elf64_getshdr(scn); 1682 if (!shdr) 1683 return -EINVAL; 1684 1685 shdr->sh_name = name_off; 1686 shdr->sh_type = SHT_PROGBITS; 1687 shdr->sh_flags = 0; 1688 shdr->sh_size = raw_sz; 1689 shdr->sh_link = 0; 1690 shdr->sh_info = 0; 1691 shdr->sh_addralign = align; 1692 shdr->sh_entsize = 0; 1693 1694 data->d_type = ELF_T_BYTE; 1695 data->d_size = raw_sz; 1696 data->d_buf = (void *)raw_data; 1697 data->d_align = align; 1698 data->d_off = 0; 1699 1700 return 0; 1701 } 1702 1703 static int finalize_btf(struct bpf_linker *linker) 1704 { 1705 struct btf *btf = linker->btf; 1706 const void *raw_data; 1707 int i, j, id, err; 1708 __u32 raw_sz; 1709 1710 /* bail out if no BTF data was produced */ 1711 if (btf__get_nr_types(linker->btf) == 0) 1712 return 0; 1713 1714 for (i = 1; i < linker->sec_cnt; i++) { 1715 struct dst_sec *sec = &linker->secs[i]; 1716 1717 if (!sec->sec_var_cnt) 1718 continue; 1719 1720 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz); 1721 if (id < 0) { 1722 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n", 1723 sec->sec_name, id); 1724 return id; 1725 } 1726 1727 for (j = 0; j < sec->sec_var_cnt; j++) { 1728 struct btf_var_secinfo *vi = &sec->sec_vars[j]; 1729 1730 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size)) 1731 return -EINVAL; 1732 } 1733 } 1734 1735 err = finalize_btf_ext(linker); 1736 if (err) { 1737 pr_warn(".BTF.ext generation failed: %d\n", err); 1738 return err; 1739 } 1740 1741 err = btf__dedup(linker->btf, linker->btf_ext, NULL); 1742 if (err) { 1743 pr_warn("BTF dedup failed: %d\n", err); 1744 return err; 1745 } 1746 1747 /* Emit .BTF section */ 1748 raw_data = btf__get_raw_data(linker->btf, &raw_sz); 1749 if (!raw_data) 1750 return -ENOMEM; 1751 1752 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz); 1753 if (err) { 1754 pr_warn("failed to write out .BTF ELF section: %d\n", err); 1755 return err; 1756 } 1757 1758 /* Emit .BTF.ext section */ 1759 if (linker->btf_ext) { 1760 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz); 1761 if (!raw_data) 1762 return -ENOMEM; 1763 1764 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz); 1765 if (err) { 1766 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err); 1767 return err; 1768 } 1769 } 1770 1771 return 0; 1772 } 1773 1774 static int emit_btf_ext_data(struct bpf_linker *linker, void *output, 1775 const char *sec_name, struct btf_ext_sec_data *sec_data) 1776 { 1777 struct btf_ext_info_sec *sec_info; 1778 void *cur = output; 1779 int str_off; 1780 size_t sz; 1781 1782 if (!sec_data->rec_cnt) 1783 return 0; 1784 1785 str_off = btf__add_str(linker->btf, sec_name); 1786 if (str_off < 0) 1787 return -ENOMEM; 1788 1789 sec_info = cur; 1790 sec_info->sec_name_off = str_off; 1791 sec_info->num_info = sec_data->rec_cnt; 1792 cur += sizeof(struct btf_ext_info_sec); 1793 1794 sz = sec_data->rec_cnt * sec_data->rec_sz; 1795 memcpy(cur, sec_data->recs, sz); 1796 cur += sz; 1797 1798 return cur - output; 1799 } 1800 1801 static int finalize_btf_ext(struct bpf_linker *linker) 1802 { 1803 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0; 1804 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0; 1805 struct btf_ext_header *hdr; 1806 void *data, *cur; 1807 int i, err, sz; 1808 1809 /* validate that all sections have the same .BTF.ext record sizes 1810 * and calculate total data size for each type of data (func info, 1811 * line info, core relos) 1812 */ 1813 for (i = 1; i < linker->sec_cnt; i++) { 1814 struct dst_sec *sec = &linker->secs[i]; 1815 1816 if (sec->func_info.rec_cnt) { 1817 if (func_rec_sz == 0) 1818 func_rec_sz = sec->func_info.rec_sz; 1819 if (func_rec_sz != sec->func_info.rec_sz) { 1820 pr_warn("mismatch in func_info record size %zu != %u\n", 1821 func_rec_sz, sec->func_info.rec_sz); 1822 return -EINVAL; 1823 } 1824 1825 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt; 1826 } 1827 if (sec->line_info.rec_cnt) { 1828 if (line_rec_sz == 0) 1829 line_rec_sz = sec->line_info.rec_sz; 1830 if (line_rec_sz != sec->line_info.rec_sz) { 1831 pr_warn("mismatch in line_info record size %zu != %u\n", 1832 line_rec_sz, sec->line_info.rec_sz); 1833 return -EINVAL; 1834 } 1835 1836 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt; 1837 } 1838 if (sec->core_relo_info.rec_cnt) { 1839 if (core_relo_rec_sz == 0) 1840 core_relo_rec_sz = sec->core_relo_info.rec_sz; 1841 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) { 1842 pr_warn("mismatch in core_relo_info record size %zu != %u\n", 1843 core_relo_rec_sz, sec->core_relo_info.rec_sz); 1844 return -EINVAL; 1845 } 1846 1847 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt; 1848 } 1849 } 1850 1851 if (!funcs_sz && !lines_sz && !core_relos_sz) 1852 return 0; 1853 1854 total_sz += sizeof(struct btf_ext_header); 1855 if (funcs_sz) { 1856 funcs_sz += sizeof(__u32); /* record size prefix */ 1857 total_sz += funcs_sz; 1858 } 1859 if (lines_sz) { 1860 lines_sz += sizeof(__u32); /* record size prefix */ 1861 total_sz += lines_sz; 1862 } 1863 if (core_relos_sz) { 1864 core_relos_sz += sizeof(__u32); /* record size prefix */ 1865 total_sz += core_relos_sz; 1866 } 1867 1868 cur = data = calloc(1, total_sz); 1869 if (!data) 1870 return -ENOMEM; 1871 1872 hdr = cur; 1873 hdr->magic = BTF_MAGIC; 1874 hdr->version = BTF_VERSION; 1875 hdr->flags = 0; 1876 hdr->hdr_len = sizeof(struct btf_ext_header); 1877 cur += sizeof(struct btf_ext_header); 1878 1879 /* All offsets are in bytes relative to the end of this header */ 1880 hdr->func_info_off = 0; 1881 hdr->func_info_len = funcs_sz; 1882 hdr->line_info_off = funcs_sz; 1883 hdr->line_info_len = lines_sz; 1884 hdr->core_relo_off = funcs_sz + lines_sz;; 1885 hdr->core_relo_len = core_relos_sz; 1886 1887 if (funcs_sz) { 1888 *(__u32 *)cur = func_rec_sz; 1889 cur += sizeof(__u32); 1890 1891 for (i = 1; i < linker->sec_cnt; i++) { 1892 struct dst_sec *sec = &linker->secs[i]; 1893 1894 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info); 1895 if (sz < 0) 1896 return sz; 1897 1898 cur += sz; 1899 } 1900 } 1901 1902 if (lines_sz) { 1903 *(__u32 *)cur = line_rec_sz; 1904 cur += sizeof(__u32); 1905 1906 for (i = 1; i < linker->sec_cnt; i++) { 1907 struct dst_sec *sec = &linker->secs[i]; 1908 1909 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info); 1910 if (sz < 0) 1911 return sz; 1912 1913 cur += sz; 1914 } 1915 } 1916 1917 if (core_relos_sz) { 1918 *(__u32 *)cur = core_relo_rec_sz; 1919 cur += sizeof(__u32); 1920 1921 for (i = 1; i < linker->sec_cnt; i++) { 1922 struct dst_sec *sec = &linker->secs[i]; 1923 1924 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info); 1925 if (sz < 0) 1926 return sz; 1927 1928 cur += sz; 1929 } 1930 } 1931 1932 linker->btf_ext = btf_ext__new(data, total_sz); 1933 err = libbpf_get_error(linker->btf_ext); 1934 if (err) { 1935 linker->btf_ext = NULL; 1936 pr_warn("failed to parse final .BTF.ext data: %d\n", err); 1937 return err; 1938 } 1939 1940 return 0; 1941 } 1942