Lines Matching +full:force +full:- +full:m1
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
158 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
189 free(linker->filename); in bpf_linker__free()
191 if (linker->elf) in bpf_linker__free()
192 elf_end(linker->elf); in bpf_linker__free()
194 if (linker->fd >= 0) in bpf_linker__free()
195 close(linker->fd); in bpf_linker__free()
197 strset__free(linker->strtab_strs); in bpf_linker__free()
199 btf__free(linker->btf); in bpf_linker__free()
200 btf_ext__free(linker->btf_ext); in bpf_linker__free()
202 for (i = 1; i < linker->sec_cnt; i++) { in bpf_linker__free()
203 struct dst_sec *sec = &linker->secs[i]; in bpf_linker__free()
205 free(sec->sec_name); in bpf_linker__free()
206 free(sec->raw_data); in bpf_linker__free()
207 free(sec->sec_vars); in bpf_linker__free()
209 free(sec->func_info.recs); in bpf_linker__free()
210 free(sec->line_info.recs); in bpf_linker__free()
211 free(sec->core_relo_info.recs); in bpf_linker__free()
213 free(linker->secs); in bpf_linker__free()
215 free(linker->glob_syms); in bpf_linker__free()
236 linker->fd = -1; in bpf_linker__new()
246 return errno = -err, NULL; in bpf_linker__new()
251 struct dst_sec *secs = linker->secs, *sec; in add_dst_sec()
252 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2; in add_dst_sec()
259 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs)); in add_dst_sec()
261 linker->secs = secs; in add_dst_sec()
262 linker->sec_cnt = new_cnt; in add_dst_sec()
264 sec = &linker->secs[new_cnt - 1]; in add_dst_sec()
265 sec->id = new_cnt - 1; in add_dst_sec()
266 sec->sec_name = strdup(sec_name); in add_dst_sec()
267 if (!sec->sec_name) in add_dst_sec()
275 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; in add_new_sym()
277 size_t sym_cnt = symtab->sec_sz / sizeof(*sym); in add_new_sym()
279 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym)); in add_new_sym()
286 symtab->raw_data = syms; in add_new_sym()
287 symtab->sec_sz += sizeof(*sym); in add_new_sym()
288 symtab->shdr->sh_size += sizeof(*sym); in add_new_sym()
289 symtab->data->d_size += sizeof(*sym); in add_new_sym()
303 linker->filename = strdup(file); in init_output_elf()
304 if (!linker->filename) in init_output_elf()
305 return -ENOMEM; in init_output_elf()
307 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); in init_output_elf()
308 if (linker->fd < 0) { in init_output_elf()
309 err = -errno; in init_output_elf()
314 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL); in init_output_elf()
315 if (!linker->elf) { in init_output_elf()
317 return -EINVAL; in init_output_elf()
321 linker->elf_hdr = elf64_newehdr(linker->elf); in init_output_elf()
322 if (!linker->elf_hdr) { in init_output_elf()
324 return -EINVAL; in init_output_elf()
327 linker->elf_hdr->e_machine = EM_BPF; in init_output_elf()
328 linker->elf_hdr->e_type = ET_REL; in init_output_elf()
330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE; in init_output_elf()
334 linker->strtab_strs = strset__new(INT_MAX, "", sizeof("")); in init_output_elf()
335 if (libbpf_get_error(linker->strtab_strs)) in init_output_elf()
336 return libbpf_get_error(linker->strtab_strs); in init_output_elf()
340 return -ENOMEM; in init_output_elf()
342 sec->scn = elf_newscn(linker->elf); in init_output_elf()
343 if (!sec->scn) { in init_output_elf()
345 return -EINVAL; in init_output_elf()
348 sec->shdr = elf64_getshdr(sec->scn); in init_output_elf()
349 if (!sec->shdr) in init_output_elf()
350 return -EINVAL; in init_output_elf()
352 sec->data = elf_newdata(sec->scn); in init_output_elf()
353 if (!sec->data) { in init_output_elf()
355 return -EINVAL; in init_output_elf()
358 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); in init_output_elf()
362 sec->sec_idx = elf_ndxscn(sec->scn); in init_output_elf()
363 linker->elf_hdr->e_shstrndx = sec->sec_idx; in init_output_elf()
364 linker->strtab_sec_idx = sec->sec_idx; in init_output_elf()
366 sec->shdr->sh_name = str_off; in init_output_elf()
367 sec->shdr->sh_type = SHT_STRTAB; in init_output_elf()
368 sec->shdr->sh_flags = SHF_STRINGS; in init_output_elf()
369 sec->shdr->sh_offset = 0; in init_output_elf()
370 sec->shdr->sh_link = 0; in init_output_elf()
371 sec->shdr->sh_info = 0; in init_output_elf()
372 sec->shdr->sh_addralign = 1; in init_output_elf()
373 sec->shdr->sh_size = sec->sec_sz = 0; in init_output_elf()
374 sec->shdr->sh_entsize = 0; in init_output_elf()
379 return -ENOMEM; in init_output_elf()
381 sec->scn = elf_newscn(linker->elf); in init_output_elf()
382 if (!sec->scn) { in init_output_elf()
384 return -EINVAL; in init_output_elf()
387 sec->shdr = elf64_getshdr(sec->scn); in init_output_elf()
388 if (!sec->shdr) in init_output_elf()
389 return -EINVAL; in init_output_elf()
391 sec->data = elf_newdata(sec->scn); in init_output_elf()
392 if (!sec->data) { in init_output_elf()
394 return -EINVAL; in init_output_elf()
396 /* Ensure libelf translates byte-order of symbol records */ in init_output_elf()
397 sec->data->d_type = ELF_T_SYM; in init_output_elf()
399 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); in init_output_elf()
403 sec->sec_idx = elf_ndxscn(sec->scn); in init_output_elf()
404 linker->symtab_sec_idx = sec->sec_idx; in init_output_elf()
406 sec->shdr->sh_name = str_off; in init_output_elf()
407 sec->shdr->sh_type = SHT_SYMTAB; in init_output_elf()
408 sec->shdr->sh_flags = 0; in init_output_elf()
409 sec->shdr->sh_offset = 0; in init_output_elf()
410 sec->shdr->sh_link = linker->strtab_sec_idx; in init_output_elf()
414 sec->shdr->sh_info = 0; in init_output_elf()
415 sec->shdr->sh_addralign = 8; in init_output_elf()
416 sec->shdr->sh_entsize = sizeof(Elf64_Sym); in init_output_elf()
419 linker->btf = btf__new_empty(); in init_output_elf()
420 err = libbpf_get_error(linker->btf); in init_output_elf()
424 /* add the special all-zero symbol */ in init_output_elf()
427 return -EINVAL; in init_output_elf()
429 init_sym->st_name = 0; in init_output_elf()
430 init_sym->st_info = 0; in init_output_elf()
431 init_sym->st_other = 0; in init_output_elf()
432 init_sym->st_shndx = SHN_UNDEF; in init_output_elf()
433 init_sym->st_value = 0; in init_output_elf()
434 init_sym->st_size = 0; in init_output_elf()
446 return libbpf_err(-EINVAL); in bpf_linker__add_file()
448 if (!linker->elf) in bpf_linker__add_file()
449 return libbpf_err(-EINVAL); in bpf_linker__add_file()
475 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; in is_dwarf_sec_name()
480 Elf64_Shdr *shdr = sec->shdr; in is_ignored_sec()
481 const char *name = sec->sec_name; in is_ignored_sec()
484 if (shdr->sh_type == SHT_STRTAB) in is_ignored_sec()
488 if (shdr->sh_type == SHT_LLVM_ADDRSIG) in is_ignored_sec()
492 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 && in is_ignored_sec()
493 strcmp(sec->sec_name, ".text") == 0) in is_ignored_sec()
497 if (is_dwarf_sec_name(sec->sec_name)) in is_ignored_sec()
500 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { in is_ignored_sec()
501 name += sizeof(".rel") - 1; in is_ignored_sec()
517 struct src_sec *secs = obj->secs, *sec; in add_src_sec()
518 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2; in add_src_sec()
525 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs)); in add_src_sec()
527 obj->secs = secs; in add_src_sec()
528 obj->sec_cnt = new_cnt; in add_src_sec()
530 sec = &obj->secs[new_cnt - 1]; in add_src_sec()
531 sec->id = new_cnt - 1; in add_src_sec()
532 sec->sec_name = sec_name; in add_src_sec()
548 unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA]; in linker_load_obj_file()
559 obj->filename = filename; in linker_load_obj_file()
561 obj->fd = open(filename, O_RDONLY | O_CLOEXEC); in linker_load_obj_file()
562 if (obj->fd < 0) { in linker_load_obj_file()
563 err = -errno; in linker_load_obj_file()
567 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL); in linker_load_obj_file()
568 if (!obj->elf) { in linker_load_obj_file()
569 err = -errno; in linker_load_obj_file()
574 /* Sanity check ELF file high-level properties */ in linker_load_obj_file()
575 ehdr = elf64_getehdr(obj->elf); in linker_load_obj_file()
577 err = -errno; in linker_load_obj_file()
583 obj_byteorder = ehdr->e_ident[EI_DATA]; in linker_load_obj_file()
585 err = -EOPNOTSUPP; in linker_load_obj_file()
590 linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder; in linker_load_obj_file()
591 linker->swapped_endian = obj_byteorder != host_byteorder; in linker_load_obj_file()
592 pr_debug("linker: set %s-endian output byte order\n", in linker_load_obj_file()
595 err = -EOPNOTSUPP; in linker_load_obj_file()
600 if (ehdr->e_type != ET_REL in linker_load_obj_file()
601 || ehdr->e_machine != EM_BPF in linker_load_obj_file()
602 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) { in linker_load_obj_file()
603 err = -EOPNOTSUPP; in linker_load_obj_file()
608 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) { in linker_load_obj_file()
609 err = -errno; in linker_load_obj_file()
615 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) { in linker_load_obj_file()
621 err = -errno; in linker_load_obj_file()
627 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name); in linker_load_obj_file()
629 err = -errno; in linker_load_obj_file()
637 err = -errno; in linker_load_obj_file()
645 return -ENOMEM; in linker_load_obj_file()
647 sec->scn = scn; in linker_load_obj_file()
648 sec->shdr = shdr; in linker_load_obj_file()
649 sec->data = data; in linker_load_obj_file()
650 sec->sec_idx = elf_ndxscn(scn); in linker_load_obj_file()
653 sec->skipped = true; in linker_load_obj_file()
657 switch (shdr->sh_type) { in linker_load_obj_file()
659 if (obj->symtab_sec_idx) { in linker_load_obj_file()
660 err = -EOPNOTSUPP; in linker_load_obj_file()
664 obj->symtab_sec_idx = sec_idx; in linker_load_obj_file()
671 obj->btf = btf__new(data->d_buf, shdr->sh_size); in linker_load_obj_file()
672 err = libbpf_get_error(obj->btf); in linker_load_obj_file()
678 sec->skipped = true; in linker_load_obj_file()
682 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size); in linker_load_obj_file()
683 err = libbpf_get_error(obj->btf_ext); in linker_load_obj_file()
689 sec->skipped = true; in linker_load_obj_file()
704 err = -EINVAL; in linker_load_obj_file()
722 if (!obj->symtab_sec_idx) { in linker_sanity_check_elf()
723 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename); in linker_sanity_check_elf()
724 return -EINVAL; in linker_sanity_check_elf()
726 if (!obj->shstrs_sec_idx) { in linker_sanity_check_elf()
727 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename); in linker_sanity_check_elf()
728 return -EINVAL; in linker_sanity_check_elf()
731 for (i = 1; i < obj->sec_cnt; i++) { in linker_sanity_check_elf()
732 sec = &obj->secs[i]; in linker_sanity_check_elf()
734 if (sec->sec_name[0] == '\0') { in linker_sanity_check_elf()
735 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename); in linker_sanity_check_elf()
736 return -EINVAL; in linker_sanity_check_elf()
739 if (is_dwarf_sec_name(sec->sec_name)) in linker_sanity_check_elf()
742 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) { in linker_sanity_check_elf()
743 pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n", in linker_sanity_check_elf()
744 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign, in linker_sanity_check_elf()
745 obj->filename); in linker_sanity_check_elf()
746 return -EINVAL; in linker_sanity_check_elf()
748 if (sec->shdr->sh_addralign != sec->data->d_align) { in linker_sanity_check_elf()
750 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign, in linker_sanity_check_elf()
751 (long long unsigned)sec->data->d_align, obj->filename); in linker_sanity_check_elf()
752 return -EINVAL; in linker_sanity_check_elf()
755 if (sec->shdr->sh_size != sec->data->d_size) { in linker_sanity_check_elf()
757 sec->sec_idx, (long long unsigned)sec->shdr->sh_size, in linker_sanity_check_elf()
758 (long long unsigned)sec->data->d_size, obj->filename); in linker_sanity_check_elf()
759 return -EINVAL; in linker_sanity_check_elf()
762 switch (sec->shdr->sh_type) { in linker_sanity_check_elf()
771 if (sec->shdr->sh_flags & SHF_EXECINSTR) { in linker_sanity_check_elf()
772 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) { in linker_sanity_check_elf()
774 sec->sec_idx, (long long unsigned)sec->shdr->sh_size, in linker_sanity_check_elf()
775 obj->filename); in linker_sanity_check_elf()
776 return -EINVAL; in linker_sanity_check_elf()
791 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename); in linker_sanity_check_elf()
792 return -EINVAL; in linker_sanity_check_elf()
805 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym)) in linker_sanity_check_elf_symtab()
806 return -EINVAL; in linker_sanity_check_elf_symtab()
807 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) in linker_sanity_check_elf_symtab()
808 return -EINVAL; in linker_sanity_check_elf_symtab()
810 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) { in linker_sanity_check_elf_symtab()
812 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); in linker_sanity_check_elf_symtab()
813 return -EINVAL; in linker_sanity_check_elf_symtab()
815 link_sec = &obj->secs[sec->shdr->sh_link]; in linker_sanity_check_elf_symtab()
816 if (link_sec->shdr->sh_type != SHT_STRTAB) { in linker_sanity_check_elf_symtab()
818 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); in linker_sanity_check_elf_symtab()
819 return -EINVAL; in linker_sanity_check_elf_symtab()
822 n = sec->shdr->sh_size / sec->shdr->sh_entsize; in linker_sanity_check_elf_symtab()
823 sym = sec->data->d_buf; in linker_sanity_check_elf_symtab()
825 int sym_type = ELF64_ST_TYPE(sym->st_info); in linker_sanity_check_elf_symtab()
826 int sym_bind = ELF64_ST_BIND(sym->st_info); in linker_sanity_check_elf_symtab()
827 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other); in linker_sanity_check_elf_symtab()
830 if (sym->st_name != 0 || sym->st_info != 0 in linker_sanity_check_elf_symtab()
831 || sym->st_other != 0 || sym->st_shndx != 0 in linker_sanity_check_elf_symtab()
832 || sym->st_value != 0 || sym->st_size != 0) { in linker_sanity_check_elf_symtab()
833 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename); in linker_sanity_check_elf_symtab()
834 return -EINVAL; in linker_sanity_check_elf_symtab()
840 i, sec->sec_idx, sym_bind); in linker_sanity_check_elf_symtab()
841 return -EINVAL; in linker_sanity_check_elf_symtab()
845 i, sec->sec_idx, sym_vis); in linker_sanity_check_elf_symtab()
846 return -EINVAL; in linker_sanity_check_elf_symtab()
848 if (sym->st_shndx == 0) { in linker_sanity_check_elf_symtab()
850 || sym->st_value != 0 || sym->st_size != 0) { in linker_sanity_check_elf_symtab()
852 i, obj->filename); in linker_sanity_check_elf_symtab()
854 return -EINVAL; in linker_sanity_check_elf_symtab()
858 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) { in linker_sanity_check_elf_symtab()
860 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename); in linker_sanity_check_elf_symtab()
861 return -EINVAL; in linker_sanity_check_elf_symtab()
864 if (sym->st_value != 0) in linker_sanity_check_elf_symtab()
865 return -EINVAL; in linker_sanity_check_elf_symtab()
879 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel)) in linker_sanity_check_elf_relos()
880 return -EINVAL; in linker_sanity_check_elf_relos()
881 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) in linker_sanity_check_elf_relos()
882 return -EINVAL; in linker_sanity_check_elf_relos()
885 if (sec->shdr->sh_link != obj->symtab_sec_idx) { in linker_sanity_check_elf_relos()
887 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); in linker_sanity_check_elf_relos()
888 return -EINVAL; in linker_sanity_check_elf_relos()
892 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) { in linker_sanity_check_elf_relos()
894 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); in linker_sanity_check_elf_relos()
895 return -EINVAL; in linker_sanity_check_elf_relos()
897 link_sec = &obj->secs[sec->shdr->sh_info]; in linker_sanity_check_elf_relos()
899 /* .rel<secname> -> <secname> pattern is followed */ in linker_sanity_check_elf_relos()
900 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0 in linker_sanity_check_elf_relos()
901 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) { in linker_sanity_check_elf_relos()
903 sec->sec_idx, obj->filename); in linker_sanity_check_elf_relos()
904 return -EINVAL; in linker_sanity_check_elf_relos()
908 if (link_sec->skipped) in linker_sanity_check_elf_relos()
912 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) { in linker_sanity_check_elf_relos()
914 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); in linker_sanity_check_elf_relos()
915 return -EINVAL; in linker_sanity_check_elf_relos()
919 n = sec->shdr->sh_size / sec->shdr->sh_entsize; in linker_sanity_check_elf_relos()
920 relo = sec->data->d_buf; in linker_sanity_check_elf_relos()
921 sym_sec = &obj->secs[obj->symtab_sec_idx]; in linker_sanity_check_elf_relos()
923 size_t sym_idx = ELF64_R_SYM(relo->r_info); in linker_sanity_check_elf_relos()
924 size_t sym_type = ELF64_R_TYPE(relo->r_info); in linker_sanity_check_elf_relos()
929 i, sec->sec_idx, sym_type, obj->filename); in linker_sanity_check_elf_relos()
930 return -EINVAL; in linker_sanity_check_elf_relos()
933 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) { in linker_sanity_check_elf_relos()
935 i, sec->sec_idx, sym_idx, obj->filename); in linker_sanity_check_elf_relos()
936 return -EINVAL; in linker_sanity_check_elf_relos()
939 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) { in linker_sanity_check_elf_relos()
940 if (relo->r_offset % sizeof(struct bpf_insn) != 0) { in linker_sanity_check_elf_relos()
942 i, sec->sec_idx, sym_idx, obj->filename); in linker_sanity_check_elf_relos()
943 return -EINVAL; in linker_sanity_check_elf_relos()
956 return -EINVAL; in check_btf_type_id()
969 return -EINVAL; in check_btf_str_off()
979 if (!obj->btf) in linker_sanity_check_btf()
982 n = btf__type_cnt(obj->btf); in linker_sanity_check_btf()
987 t = btf_type_by_id(obj->btf, i); in linker_sanity_check_btf()
994 return -EINVAL; in linker_sanity_check_btf()
1001 if (!btf__str_by_offset(obj->btf, *str_off)) in linker_sanity_check_btf()
1002 return -EINVAL; in linker_sanity_check_btf()
1013 if (!obj->btf_ext) in linker_sanity_check_btf_ext()
1017 if (!obj->btf) in linker_sanity_check_btf_ext()
1018 return -EINVAL; in linker_sanity_check_btf_ext()
1020 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf); in linker_sanity_check_btf_ext()
1021 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf); in linker_sanity_check_btf_ext()
1035 dst_sec->sec_sz = 0; in init_sec()
1036 dst_sec->sec_idx = 0; in init_sec()
1037 dst_sec->ephemeral = src_sec->ephemeral; in init_sec()
1040 if (src_sec->ephemeral) in init_sec()
1043 scn = elf_newscn(linker->elf); in init_sec()
1045 return -ENOMEM; in init_sec()
1048 return -ENOMEM; in init_sec()
1051 return -ENOMEM; in init_sec()
1053 dst_sec->scn = scn; in init_sec()
1054 dst_sec->shdr = shdr; in init_sec()
1055 dst_sec->data = data; in init_sec()
1056 dst_sec->sec_idx = elf_ndxscn(scn); in init_sec()
1058 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name); in init_sec()
1062 shdr->sh_name = name_off; in init_sec()
1063 shdr->sh_type = src_sec->shdr->sh_type; in init_sec()
1064 shdr->sh_flags = src_sec->shdr->sh_flags; in init_sec()
1065 shdr->sh_size = 0; in init_sec()
1070 shdr->sh_link = 0; in init_sec()
1071 shdr->sh_info = 0; in init_sec()
1072 shdr->sh_addralign = src_sec->shdr->sh_addralign; in init_sec()
1073 shdr->sh_entsize = src_sec->shdr->sh_entsize; in init_sec()
1075 data->d_type = src_sec->data->d_type; in init_sec()
1076 data->d_size = 0; in init_sec()
1077 data->d_buf = NULL; in init_sec()
1078 data->d_align = src_sec->data->d_align; in init_sec()
1079 data->d_off = 0; in init_sec()
1089 for (i = 1; i < linker->sec_cnt; i++) { in find_dst_sec_by_name()
1090 sec = &linker->secs[i]; in find_dst_sec_by_name()
1092 if (strcmp(sec->sec_name, sec_name) == 0) in find_dst_sec_by_name()
1101 if (dst->ephemeral || src->ephemeral) in secs_match()
1104 if (dst->shdr->sh_type != src->shdr->sh_type) { in secs_match()
1105 pr_warn("sec %s types mismatch\n", dst->sec_name); in secs_match()
1108 if (dst->shdr->sh_flags != src->shdr->sh_flags) { in secs_match()
1109 pr_warn("sec %s flags mismatch\n", dst->sec_name); in secs_match()
1112 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) { in secs_match()
1113 pr_warn("sec %s entsize mismatch\n", dst->sec_name); in secs_match()
1122 if (dst_sec->sec_sz != src_sec->shdr->sh_size) in sec_content_is_same()
1124 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0) in sec_content_is_same()
1131 if (!sec || sec->ephemeral) in is_exec_sec()
1133 return (sec->shdr->sh_type == SHT_PROGBITS) && in is_exec_sec()
1134 (sec->shdr->sh_flags & SHF_EXECINSTR); in is_exec_sec()
1157 if (src->ephemeral) in extend_sec()
1161 * ephemeral) and non-externs (map definitions). So it's possible that in extend_sec()
1162 * it has to be "upgraded" from ephemeral to non-ephemeral when the in extend_sec()
1163 * first non-ephemeral entity appears. In such case, we add ELF in extend_sec()
1166 if (dst->ephemeral) { in extend_sec()
1172 dst_align = dst->shdr->sh_addralign; in extend_sec()
1173 src_align = src->shdr->sh_addralign; in extend_sec()
1179 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align; in extend_sec()
1181 /* no need to re-align final size */ in extend_sec()
1182 dst_final_sz = dst_align_sz + src->shdr->sh_size; in extend_sec()
1184 if (src->shdr->sh_type != SHT_NOBITS) { in extend_sec()
1185 tmp = realloc(dst->raw_data, dst_final_sz); in extend_sec()
1187 * 1. When dst->raw_data is NULL it returns: in extend_sec()
1189 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL, in extend_sec()
1194 * dst->raw_data would be freed again in bpf_linker__free(). in extend_sec()
1199 return -ENOMEM; in extend_sec()
1200 dst->raw_data = tmp; in extend_sec()
1203 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz); in extend_sec()
1205 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size); in extend_sec()
1207 /* convert added bpf insns to native byte-order */ in extend_sec()
1208 if (linker->swapped_endian && is_exec_sec(dst)) in extend_sec()
1209 exec_sec_bswap(dst->raw_data + dst_align_sz, src->shdr->sh_size); in extend_sec()
1212 dst->sec_sz = dst_final_sz; in extend_sec()
1213 dst->shdr->sh_size = dst_final_sz; in extend_sec()
1214 dst->data->d_size = dst_final_sz; in extend_sec()
1216 dst->shdr->sh_addralign = dst_align; in extend_sec()
1217 dst->data->d_align = dst_align; in extend_sec()
1219 src->dst_off = dst_align_sz; in extend_sec()
1226 if (!sec || sec->skipped) in is_data_sec()
1229 if (sec->ephemeral) in is_data_sec()
1231 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS; in is_data_sec()
1236 if (!sec || sec->skipped || sec->ephemeral) in is_relo_sec()
1238 return sec->shdr->sh_type == SHT_REL; in is_relo_sec()
1245 for (i = 1; i < obj->sec_cnt; i++) { in linker_append_sec_data()
1249 src_sec = &obj->secs[i]; in linker_append_sec_data()
1253 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); in linker_append_sec_data()
1255 dst_sec = add_dst_sec(linker, src_sec->sec_name); in linker_append_sec_data()
1257 return -ENOMEM; in linker_append_sec_data()
1260 pr_warn("failed to init section '%s'\n", src_sec->sec_name); in linker_append_sec_data()
1265 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name); in linker_append_sec_data()
1266 return -1; in linker_append_sec_data()
1270 if (strcmp(src_sec->sec_name, "license") == 0 in linker_append_sec_data()
1271 || strcmp(src_sec->sec_name, "version") == 0) { in linker_append_sec_data()
1273 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name); in linker_append_sec_data()
1274 return -EINVAL; in linker_append_sec_data()
1276 src_sec->skipped = true; in linker_append_sec_data()
1277 src_sec->dst_id = dst_sec->id; in linker_append_sec_data()
1283 src_sec->dst_id = dst_sec->id; in linker_append_sec_data()
1295 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; in linker_append_elf_syms()
1296 Elf64_Sym *sym = symtab->data->d_buf; in linker_append_elf_syms()
1297 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err; in linker_append_elf_syms()
1298 int str_sec_idx = symtab->shdr->sh_link; in linker_append_elf_syms()
1301 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map)); in linker_append_elf_syms()
1302 if (!obj->sym_map) in linker_append_elf_syms()
1303 return -ENOMEM; in linker_append_elf_syms()
1306 /* We already validated all-zero symbol #0 and we already in linker_append_elf_syms()
1312 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); in linker_append_elf_syms()
1314 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename); in linker_append_elf_syms()
1315 return -EINVAL; in linker_append_elf_syms()
1328 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; in get_sym_by_idx()
1329 Elf64_Sym *syms = symtab->raw_data; in get_sym_by_idx()
1340 for (i = 0; i < linker->glob_sym_cnt; i++) { in find_glob_sym()
1341 glob_sym = &linker->glob_syms[i]; in find_glob_sym()
1342 name = strset__data(linker->strtab_strs) + glob_sym->name_off; in find_glob_sym()
1355 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1, in add_glob_sym()
1356 sizeof(*linker->glob_syms)); in add_glob_sym()
1360 sym = &syms[linker->glob_sym_cnt]; in add_glob_sym()
1362 sym->var_idx = -1; in add_glob_sym()
1364 linker->glob_syms = syms; in add_glob_sym()
1365 linker->glob_sym_cnt++; in add_glob_sym()
1386 n1 = btf__str_by_offset(btf1, t1->name_off); in glob_sym_btf_matches()
1387 n2 = btf__str_by_offset(btf2, t2->name_off); in glob_sym_btf_matches()
1426 n1 = btf__str_by_offset(btf1, t1->name_off); in glob_sym_btf_matches()
1427 n2 = btf__str_by_offset(btf2, t2->name_off); in glob_sym_btf_matches()
1447 if (t1->size != t2->size) { in glob_sym_btf_matches()
1449 sym_name, btf_kind_str(t1), n1, t1->size, t2->size); in glob_sym_btf_matches()
1459 id1 = t1->type; in glob_sym_btf_matches()
1460 id2 = t2->type; in glob_sym_btf_matches()
1464 id1 = btf_array(t1)->type; in glob_sym_btf_matches()
1465 id2 = btf_array(t2)->type; in glob_sym_btf_matches()
1476 id1 = t1->type; in glob_sym_btf_matches()
1477 id2 = t2->type; in glob_sym_btf_matches()
1481 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC; in glob_sym_btf_matches()
1482 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC; in glob_sym_btf_matches()
1488 id1 = t1->type; in glob_sym_btf_matches()
1489 id2 = t2->type; in glob_sym_btf_matches()
1493 const struct btf_member *m1, *m2; in glob_sym_btf_matches() local
1505 m1 = btf_members(t1); in glob_sym_btf_matches()
1507 for (i = 0; i < n; i++, m1++, m2++) { in glob_sym_btf_matches()
1508 n1 = btf__str_by_offset(btf1, m1->name_off); in glob_sym_btf_matches()
1509 n2 = btf__str_by_offset(btf2, m2->name_off); in glob_sym_btf_matches()
1515 if (m1->offset != m2->offset) { in glob_sym_btf_matches()
1520 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type)) in glob_sym_btf_matches()
1527 const struct btf_param *m1, *m2; in glob_sym_btf_matches() local
1536 m1 = btf_params(t1); in glob_sym_btf_matches()
1538 for (i = 0; i < n; i++, m1++, m2++) { in glob_sym_btf_matches()
1540 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type)) in glob_sym_btf_matches()
1545 id1 = t1->type; in glob_sym_btf_matches()
1546 id2 = t2->type; in glob_sym_btf_matches()
1574 if (main_def->map_type != extra_def->map_type) { in map_defs_match()
1580 if (main_def->key_size != extra_def->key_size) { in map_defs_match()
1584 if (!!main_def->key_type_id != !!extra_def->key_type_id) { in map_defs_match()
1588 if ((main_def->parts & MAP_DEF_KEY_TYPE) in map_defs_match()
1590 main_btf, main_def->key_type_id, in map_defs_match()
1591 extra_btf, extra_def->key_type_id)) { in map_defs_match()
1597 if (main_def->value_size != extra_def->value_size) { in map_defs_match()
1601 if (!!main_def->value_type_id != !!extra_def->value_type_id) { in map_defs_match()
1605 if ((main_def->parts & MAP_DEF_VALUE_TYPE) in map_defs_match()
1607 main_btf, main_def->value_type_id, in map_defs_match()
1608 extra_btf, extra_def->value_type_id)) { in map_defs_match()
1613 if (main_def->max_entries != extra_def->max_entries) { in map_defs_match()
1617 if (main_def->map_flags != extra_def->map_flags) { in map_defs_match()
1621 if (main_def->numa_node != extra_def->numa_node) { in map_defs_match()
1625 if (main_def->pinning != extra_def->pinning) { in map_defs_match()
1630 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) { in map_defs_match()
1635 if (main_def->parts & MAP_DEF_INNER_MAP) { in map_defs_match()
1661 t = btf__type_by_id(obj->btf, btf_id); in glob_map_defs_match()
1666 t = skip_mods_and_typedefs(obj->btf, t->type, NULL); in glob_map_defs_match()
1668 err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def); in glob_map_defs_match()
1674 /* re-parse existing map definition */ in glob_map_defs_match()
1675 t = btf__type_by_id(linker->btf, glob_sym->btf_id); in glob_map_defs_match()
1676 t = skip_mods_and_typedefs(linker->btf, t->type, NULL); in glob_map_defs_match()
1677 err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def); in glob_map_defs_match()
1688 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def, in glob_map_defs_match()
1689 obj->btf, &src_def, &src_inner_def); in glob_map_defs_match()
1701 if (!glob_sym->btf_id || !btf_id) { in glob_syms_match()
1706 src_t = btf__type_by_id(obj->btf, btf_id); in glob_syms_match()
1714 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0) in glob_syms_match()
1718 linker->btf, glob_sym->btf_id, obj->btf, btf_id)) in glob_syms_match()
1726 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC) in btf_is_non_static()
1738 if (!obj->btf) { in find_glob_sym_btf()
1739 pr_warn("failed to find BTF info for object '%s'\n", obj->filename); in find_glob_sym_btf()
1740 return -EINVAL; in find_glob_sym_btf()
1743 n = btf__type_cnt(obj->btf); in find_glob_sym_btf()
1745 t = btf__type_by_id(obj->btf, i); in find_glob_sym_btf()
1751 name = btf__str_by_offset(obj->btf, t->name_off); in find_glob_sym_btf()
1765 t = btf__type_by_id(obj->btf, vi->type); in find_glob_sym_btf()
1766 name = btf__str_by_offset(obj->btf, t->name_off); in find_glob_sym_btf()
1770 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC) in find_glob_sym_btf()
1775 if (btf_id && btf_id != vi->type) { in find_glob_sym_btf()
1777 sym_name, btf_id, vi->type); in find_glob_sym_btf()
1778 return -EINVAL; in find_glob_sym_btf()
1782 *out_btf_id = vi->type; in find_glob_sym_btf()
1788 /* free-floating extern or global FUNC */ in find_glob_sym_btf()
1796 return -ENOENT; in find_glob_sym_btf()
1804 for (i = 1; i < obj->sec_cnt; i++) { in find_src_sec_by_name()
1805 sec = &obj->secs[i]; in find_src_sec_by_name()
1807 if (strcmp(sec->sec_name, sec_name) == 0) in find_src_sec_by_name()
1827 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; in complete_extern_btf_info()
1831 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0); in complete_extern_btf_info()
1834 src_t = btf_type_by_id(src_btf, src_t->type); in complete_extern_btf_info()
1835 dst_t = btf_type_by_id(dst_btf, dst_t->type); in complete_extern_btf_info()
1851 if (!src_p->name_off) in complete_extern_btf_info()
1855 s = btf__str_by_offset(src_btf, src_p->name_off); in complete_extern_btf_info()
1859 dst_p->name_off = off; in complete_extern_btf_info()
1866 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info)); in sym_update_bind()
1871 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type); in sym_update_type()
1879 sym->st_other &= ~0x03; in sym_update_visibility()
1880 sym->st_other |= sym_vis; in sym_update_visibility()
1895 sym_type = ELF64_ST_TYPE(sym->st_info); in linker_append_elf_sym()
1896 sym_bind = ELF64_ST_BIND(sym->st_info); in linker_append_elf_sym()
1897 sym_vis = ELF64_ST_VISIBILITY(sym->st_other); in linker_append_elf_sym()
1898 sym_is_extern = sym->st_shndx == SHN_UNDEF; in linker_append_elf_sym()
1901 if (!obj->btf) { in linker_append_elf_sym()
1903 return -ENOTSUP; in linker_append_elf_sym()
1905 } else if (sym->st_shndx < SHN_LORESERVE) { in linker_append_elf_sym()
1906 src_sec = &obj->secs[sym->st_shndx]; in linker_append_elf_sym()
1907 if (src_sec->skipped) in linker_append_elf_sym()
1909 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_elf_sym()
1912 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) { in linker_append_elf_sym()
1913 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx; in linker_append_elf_sym()
1930 t = btf__type_by_id(obj->btf, btf_sec_id); in linker_append_elf_sym()
1931 sec_name = btf__str_by_offset(obj->btf, t->name_off); in linker_append_elf_sym()
1943 return -ENOENT; in linker_append_elf_sym()
1945 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_elf_sym()
1955 obj->sym_map[src_sym_idx] = glob_sym->sym_idx; in linker_append_elf_sym()
1957 /* If both symbols are non-externs, at least one of in linker_append_elf_sym()
1961 if (!sym_is_extern && !glob_sym->is_extern in linker_append_elf_sym()
1962 && !glob_sym->is_weak && sym_bind != STB_WEAK) { in linker_append_elf_sym()
1963 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n", in linker_append_elf_sym()
1964 src_sym_idx, sym_name, obj->filename); in linker_append_elf_sym()
1965 return -EINVAL; in linker_append_elf_sym()
1969 return -EINVAL; in linker_append_elf_sym()
1971 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx); in linker_append_elf_sym()
1973 /* If new symbol is strong, then force dst_sym to be strong as in linker_append_elf_sym()
1974 * well; this way a mix of weak and non-weak extern in linker_append_elf_sym()
1983 glob_sym->is_weak = false; in linker_append_elf_sym()
1986 /* Non-default visibility is "contaminating", with stricter in linker_append_elf_sym()
1992 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other)) in linker_append_elf_sym()
2005 * strong-strong conflict. We also already tightened binding in linker_append_elf_sym()
2008 if (!glob_sym->is_extern && sym_bind == STB_WEAK) in linker_append_elf_sym()
2011 /* At this point, new symbol is strong non-extern, in linker_append_elf_sym()
2016 dst_sym->st_shndx = dst_sec->sec_idx; in linker_append_elf_sym()
2017 dst_sym->st_value = src_sec->dst_off + sym->st_value; in linker_append_elf_sym()
2018 dst_sym->st_size = sym->st_size; in linker_append_elf_sym()
2020 /* see comment below about dst_sec->id vs dst_sec->sec_idx */ in linker_append_elf_sym()
2021 glob_sym->sec_id = dst_sec->id; in linker_append_elf_sym()
2022 glob_sym->is_extern = false; in linker_append_elf_sym()
2024 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id, in linker_append_elf_sym()
2025 obj->btf, btf_id)) in linker_append_elf_sym()
2026 return -EINVAL; in linker_append_elf_sym()
2029 glob_sym->underlying_btf_id = 0; in linker_append_elf_sym()
2031 obj->sym_map[src_sym_idx] = glob_sym->sym_idx; in linker_append_elf_sym()
2036 name_off = strset__add_str(linker->strtab_strs, sym_name); in linker_append_elf_sym()
2042 return -ENOMEM; in linker_append_elf_sym()
2044 dst_sym->st_name = name_off; in linker_append_elf_sym()
2045 dst_sym->st_info = sym->st_info; in linker_append_elf_sym()
2046 dst_sym->st_other = sym->st_other; in linker_append_elf_sym()
2047 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx; in linker_append_elf_sym()
2048 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; in linker_append_elf_sym()
2049 dst_sym->st_size = sym->st_size; in linker_append_elf_sym()
2051 obj->sym_map[src_sym_idx] = dst_sym_idx; in linker_append_elf_sym()
2054 dst_sec->sec_sym_idx = dst_sym_idx; in linker_append_elf_sym()
2055 dst_sym->st_value = 0; in linker_append_elf_sym()
2061 return -ENOMEM; in linker_append_elf_sym()
2063 glob_sym->sym_idx = dst_sym_idx; in linker_append_elf_sym()
2064 /* we use dst_sec->id (and not dst_sec->sec_idx), because in linker_append_elf_sym()
2068 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0. in linker_append_elf_sym()
2070 glob_sym->sec_id = dst_sec ? dst_sec->id : 0; in linker_append_elf_sym()
2071 glob_sym->name_off = name_off; in linker_append_elf_sym()
2073 glob_sym->btf_id = 0; in linker_append_elf_sym()
2074 glob_sym->is_extern = sym_is_extern; in linker_append_elf_sym()
2075 glob_sym->is_weak = sym_bind == STB_WEAK; in linker_append_elf_sym()
2083 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx]; in linker_append_elf_relos()
2086 for (i = 1; i < obj->sec_cnt; i++) { in linker_append_elf_relos()
2092 src_sec = &obj->secs[i]; in linker_append_elf_relos()
2096 /* shdr->sh_info points to relocatable section */ in linker_append_elf_relos()
2097 src_linked_sec = &obj->secs[src_sec->shdr->sh_info]; in linker_append_elf_relos()
2098 if (src_linked_sec->skipped) in linker_append_elf_relos()
2101 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); in linker_append_elf_relos()
2103 dst_sec = add_dst_sec(linker, src_sec->sec_name); in linker_append_elf_relos()
2105 return -ENOMEM; in linker_append_elf_relos()
2108 pr_warn("failed to init section '%s'\n", src_sec->sec_name); in linker_append_elf_relos()
2112 pr_warn("sections %s are not compatible\n", src_sec->sec_name); in linker_append_elf_relos()
2113 return -1; in linker_append_elf_relos()
2116 /* shdr->sh_link points to SYMTAB */ in linker_append_elf_relos()
2117 dst_sec->shdr->sh_link = linker->symtab_sec_idx; in linker_append_elf_relos()
2119 /* shdr->sh_info points to relocated section */ in linker_append_elf_relos()
2120 dst_linked_sec = &linker->secs[src_linked_sec->dst_id]; in linker_append_elf_relos()
2121 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx; in linker_append_elf_relos()
2123 src_sec->dst_id = dst_sec->id; in linker_append_elf_relos()
2128 src_rel = src_sec->data->d_buf; in linker_append_elf_relos()
2129 dst_rel = dst_sec->raw_data + src_sec->dst_off; in linker_append_elf_relos()
2130 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize; in linker_append_elf_relos()
2135 src_sym_idx = ELF64_R_SYM(src_rel->r_info); in linker_append_elf_relos()
2136 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx; in linker_append_elf_relos()
2138 dst_sym_idx = obj->sym_map[src_sym_idx]; in linker_append_elf_relos()
2139 dst_rel->r_offset += src_linked_sec->dst_off; in linker_append_elf_relos()
2140 sym_type = ELF64_R_TYPE(src_rel->r_info); in linker_append_elf_relos()
2141 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type); in linker_append_elf_relos()
2143 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) { in linker_append_elf_relos()
2144 struct src_sec *sec = &obj->secs[src_sym->st_shndx]; in linker_append_elf_relos()
2147 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) { in linker_append_elf_relos()
2157 insn = dst_linked_sec->raw_data + dst_rel->r_offset; in linker_append_elf_relos()
2158 if (insn->code == (BPF_JMP | BPF_CALL)) in linker_append_elf_relos()
2159 insn->imm += sec->dst_off / sizeof(struct bpf_insn); in linker_append_elf_relos()
2161 insn->imm += sec->dst_off; in linker_append_elf_relos()
2163 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n"); in linker_append_elf_relos()
2164 return -EINVAL; in linker_append_elf_relos()
2177 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; in find_sym_by_name()
2178 Elf64_Sym *sym = symtab->data->d_buf; in find_sym_by_name()
2179 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; in find_sym_by_name()
2180 int str_sec_idx = symtab->shdr->sh_link; in find_sym_by_name()
2184 if (sym->st_shndx != sec_idx) in find_sym_by_name()
2186 if (ELF64_ST_TYPE(sym->st_info) != sym_type) in find_sym_by_name()
2189 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); in find_sym_by_name()
2208 if (!obj->btf) in linker_fixup_btf()
2211 n = btf__type_cnt(obj->btf); in linker_fixup_btf()
2216 t = btf_type_by_id(obj->btf, i); in linker_fixup_btf()
2220 sec_name = btf__str_by_offset(obj->btf, t->name_off); in linker_fixup_btf()
2224 if (sec->shdr) in linker_fixup_btf()
2225 t->size = sec->shdr->sh_size; in linker_fixup_btf()
2232 * sections, we pre-create "section shells" to be able in linker_fixup_btf()
2233 * to keep track of extra per-section metadata later in linker_fixup_btf()
2242 * matching non-extern VAR/FUNC in other sections. in linker_fixup_btf()
2256 return -ENOMEM; in linker_fixup_btf()
2258 sec->ephemeral = true; in linker_fixup_btf()
2259 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */ in linker_fixup_btf()
2263 sec->sec_type_id = i; in linker_fixup_btf()
2268 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type); in linker_fixup_btf()
2277 var_name = btf__str_by_offset(obj->btf, vt->name_off); in linker_fixup_btf()
2278 var_linkage = btf_var(vt)->linkage; in linker_fixup_btf()
2284 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name); in linker_fixup_btf()
2287 return -ENOENT; in linker_fixup_btf()
2290 vi->offset = sym->st_value; in linker_fixup_btf()
2303 if (!obj->btf) in linker_append_btf()
2306 start_id = btf__type_cnt(linker->btf); in linker_append_btf()
2307 n = btf__type_cnt(obj->btf); in linker_append_btf()
2309 obj->btf_type_map = calloc(n + 1, sizeof(int)); in linker_append_btf()
2310 if (!obj->btf_type_map) in linker_append_btf()
2311 return -ENOMEM; in linker_append_btf()
2316 t = btf__type_by_id(obj->btf, i); in linker_append_btf()
2324 name = btf__str_by_offset(obj->btf, t->name_off); in linker_append_btf()
2336 * to strong symbol or weak got upgraded to non-weak in linker_append_btf()
2338 if (glob_sym->underlying_btf_id == 0) in linker_append_btf()
2339 glob_sym->underlying_btf_id = -t->type; in linker_append_btf()
2345 if (glob_sym->btf_id) { in linker_append_btf()
2347 obj->btf_type_map[i] = glob_sym->btf_id; in linker_append_btf()
2352 id = btf__add_type(linker->btf, obj->btf, t); in linker_append_btf()
2354 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename); in linker_append_btf()
2358 obj->btf_type_map[i] = id; in linker_append_btf()
2362 glob_sym->btf_id = id; in linker_append_btf()
2363 glob_sym->underlying_btf_id = -t->type; in linker_append_btf()
2368 n = btf__type_cnt(linker->btf); in linker_append_btf()
2370 struct btf_type *dst_t = btf_type_by_id(linker->btf, i); in linker_append_btf()
2379 int new_id = obj->btf_type_map[*type_id]; in linker_append_btf()
2385 return -EINVAL; in linker_append_btf()
2388 *type_id = obj->btf_type_map[*type_id]; in linker_append_btf()
2395 for (i = 0; i < linker->glob_sym_cnt; i++) { in linker_append_btf()
2396 struct glob_sym *glob_sym = &linker->glob_syms[i]; in linker_append_btf()
2399 if (glob_sym->underlying_btf_id >= 0) in linker_append_btf()
2402 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id]; in linker_append_btf()
2404 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id); in linker_append_btf()
2405 glob_t->type = glob_sym->underlying_btf_id; in linker_append_btf()
2409 for (i = 1; i < obj->sec_cnt; i++) { in linker_append_btf()
2415 src_sec = &obj->secs[i]; in linker_append_btf()
2416 if (!src_sec->sec_type_id || src_sec->skipped) in linker_append_btf()
2418 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_btf()
2424 * file otherwise has no read-only static variables in in linker_append_btf()
2428 dst_sec->has_btf = true; in linker_append_btf()
2430 t = btf__type_by_id(obj->btf, src_sec->sec_type_id); in linker_append_btf()
2434 void *sec_vars = dst_sec->sec_vars; in linker_append_btf()
2435 int new_id = obj->btf_type_map[src_var->type]; in linker_append_btf()
2438 t = btf_type_by_id(linker->btf, new_id); in linker_append_btf()
2440 name = btf__str_by_offset(linker->btf, t->name_off); in linker_append_btf()
2442 if (glob_sym->sec_id != dst_sec->id) { in linker_append_btf()
2444 name, glob_sym->sec_id, dst_sec->id); in linker_append_btf()
2445 return -EINVAL; in linker_append_btf()
2454 if (glob_sym && glob_sym->var_idx >= 0) { in linker_append_btf()
2461 dst_var = &dst_sec->sec_vars[glob_sym->var_idx]; in linker_append_btf()
2464 * re-calculate and update it in sec_var. in linker_append_btf()
2466 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id); in linker_append_btf()
2470 return -EINVAL; in linker_append_btf()
2472 dst_var->size = sz; in linker_append_btf()
2477 dst_sec->sec_var_cnt + 1, in linker_append_btf()
2478 sizeof(*dst_sec->sec_vars)); in linker_append_btf()
2480 return -ENOMEM; in linker_append_btf()
2482 dst_sec->sec_vars = sec_vars; in linker_append_btf()
2483 dst_sec->sec_var_cnt++; in linker_append_btf()
2485 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1]; in linker_append_btf()
2486 dst_var->type = obj->btf_type_map[src_var->type]; in linker_append_btf()
2487 dst_var->size = src_var->size; in linker_append_btf()
2488 dst_var->offset = src_sec->dst_off + src_var->offset; in linker_append_btf()
2491 glob_sym->var_idx = dst_sec->sec_var_cnt - 1; in linker_append_btf()
2502 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz); in add_btf_ext_rec()
2505 ext_data->recs = tmp; in add_btf_ext_rec()
2507 tmp += ext_data->rec_cnt * ext_data->rec_sz; in add_btf_ext_rec()
2508 memcpy(tmp, src_rec, ext_data->rec_sz); in add_btf_ext_rec()
2510 ext_data->rec_cnt++; in add_btf_ext_rec()
2523 if (!obj->btf_ext) in linker_append_btf_ext()
2526 rec_sz = obj->btf_ext->func_info.rec_size; in linker_append_btf_ext()
2527 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) { in linker_append_btf_ext()
2530 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); in linker_append_btf_ext()
2534 return -EINVAL; in linker_append_btf_ext()
2536 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_btf_ext()
2538 if (dst_sec->func_info.rec_sz == 0) in linker_append_btf_ext()
2539 dst_sec->func_info.rec_sz = rec_sz; in linker_append_btf_ext()
2540 if (dst_sec->func_info.rec_sz != rec_sz) { in linker_append_btf_ext()
2542 return -EINVAL; in linker_append_btf_ext()
2545 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) { in linker_append_btf_ext()
2546 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec); in linker_append_btf_ext()
2548 return -ENOMEM; in linker_append_btf_ext()
2550 dst_rec->insn_off += src_sec->dst_off; in linker_append_btf_ext()
2551 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; in linker_append_btf_ext()
2555 rec_sz = obj->btf_ext->line_info.rec_size; in linker_append_btf_ext()
2556 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) { in linker_append_btf_ext()
2559 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); in linker_append_btf_ext()
2563 return -EINVAL; in linker_append_btf_ext()
2565 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_btf_ext()
2567 if (dst_sec->line_info.rec_sz == 0) in linker_append_btf_ext()
2568 dst_sec->line_info.rec_sz = rec_sz; in linker_append_btf_ext()
2569 if (dst_sec->line_info.rec_sz != rec_sz) { in linker_append_btf_ext()
2571 return -EINVAL; in linker_append_btf_ext()
2574 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) { in linker_append_btf_ext()
2575 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec); in linker_append_btf_ext()
2577 return -ENOMEM; in linker_append_btf_ext()
2579 dst_rec->insn_off += src_sec->dst_off; in linker_append_btf_ext()
2581 s = btf__str_by_offset(obj->btf, src_rec->file_name_off); in linker_append_btf_ext()
2582 str_off = btf__add_str(linker->btf, s); in linker_append_btf_ext()
2584 return -ENOMEM; in linker_append_btf_ext()
2585 dst_rec->file_name_off = str_off; in linker_append_btf_ext()
2587 s = btf__str_by_offset(obj->btf, src_rec->line_off); in linker_append_btf_ext()
2588 str_off = btf__add_str(linker->btf, s); in linker_append_btf_ext()
2590 return -ENOMEM; in linker_append_btf_ext()
2591 dst_rec->line_off = str_off; in linker_append_btf_ext()
2593 /* dst_rec->line_col is fine */ in linker_append_btf_ext()
2597 rec_sz = obj->btf_ext->core_relo_info.rec_size; in linker_append_btf_ext()
2598 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) { in linker_append_btf_ext()
2601 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); in linker_append_btf_ext()
2605 return -EINVAL; in linker_append_btf_ext()
2607 dst_sec = &linker->secs[src_sec->dst_id]; in linker_append_btf_ext()
2609 if (dst_sec->core_relo_info.rec_sz == 0) in linker_append_btf_ext()
2610 dst_sec->core_relo_info.rec_sz = rec_sz; in linker_append_btf_ext()
2611 if (dst_sec->core_relo_info.rec_sz != rec_sz) { in linker_append_btf_ext()
2613 return -EINVAL; in linker_append_btf_ext()
2616 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) { in linker_append_btf_ext()
2617 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec); in linker_append_btf_ext()
2619 return -ENOMEM; in linker_append_btf_ext()
2621 dst_rec->insn_off += src_sec->dst_off; in linker_append_btf_ext()
2622 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; in linker_append_btf_ext()
2624 s = btf__str_by_offset(obj->btf, src_rec->access_str_off); in linker_append_btf_ext()
2625 str_off = btf__add_str(linker->btf, s); in linker_append_btf_ext()
2627 return -ENOMEM; in linker_append_btf_ext()
2628 dst_rec->access_str_off = str_off; in linker_append_btf_ext()
2630 /* dst_rec->kind is fine */ in linker_append_btf_ext()
2644 if (!linker->elf) in bpf_linker__finalize()
2645 return libbpf_err(-EINVAL); in bpf_linker__finalize()
2652 strs_sz = strset__data_size(linker->strtab_strs); in bpf_linker__finalize()
2653 strs = strset__data(linker->strtab_strs); in bpf_linker__finalize()
2655 sec = &linker->secs[linker->strtab_sec_idx]; in bpf_linker__finalize()
2656 sec->data->d_align = 1; in bpf_linker__finalize()
2657 sec->data->d_off = 0LL; in bpf_linker__finalize()
2658 sec->data->d_buf = (void *)strs; in bpf_linker__finalize()
2659 sec->data->d_type = ELF_T_BYTE; in bpf_linker__finalize()
2660 sec->data->d_size = strs_sz; in bpf_linker__finalize()
2661 sec->shdr->sh_size = strs_sz; in bpf_linker__finalize()
2663 for (i = 1; i < linker->sec_cnt; i++) { in bpf_linker__finalize()
2664 sec = &linker->secs[i]; in bpf_linker__finalize()
2667 if (sec->sec_idx == linker->strtab_sec_idx) in bpf_linker__finalize()
2671 if (!sec->scn) in bpf_linker__finalize()
2674 /* restore sections with bpf insns to target byte-order */ in bpf_linker__finalize()
2675 if (linker->swapped_endian && is_exec_sec(sec)) in bpf_linker__finalize()
2676 exec_sec_bswap(sec->raw_data, sec->sec_sz); in bpf_linker__finalize()
2678 sec->data->d_buf = sec->raw_data; in bpf_linker__finalize()
2682 if (elf_update(linker->elf, ELF_C_NULL) < 0) { in bpf_linker__finalize()
2683 err = -errno; in bpf_linker__finalize()
2689 if (elf_update(linker->elf, ELF_C_WRITE) < 0) { in bpf_linker__finalize()
2690 err = -errno; in bpf_linker__finalize()
2695 elf_end(linker->elf); in bpf_linker__finalize()
2696 close(linker->fd); in bpf_linker__finalize()
2698 linker->elf = NULL; in bpf_linker__finalize()
2699 linker->fd = -1; in bpf_linker__finalize()
2712 name_off = strset__add_str(linker->strtab_strs, sec_name); in emit_elf_data_sec()
2716 scn = elf_newscn(linker->elf); in emit_elf_data_sec()
2718 return -ENOMEM; in emit_elf_data_sec()
2721 return -ENOMEM; in emit_elf_data_sec()
2724 return -EINVAL; in emit_elf_data_sec()
2726 shdr->sh_name = name_off; in emit_elf_data_sec()
2727 shdr->sh_type = SHT_PROGBITS; in emit_elf_data_sec()
2728 shdr->sh_flags = 0; in emit_elf_data_sec()
2729 shdr->sh_size = raw_sz; in emit_elf_data_sec()
2730 shdr->sh_link = 0; in emit_elf_data_sec()
2731 shdr->sh_info = 0; in emit_elf_data_sec()
2732 shdr->sh_addralign = align; in emit_elf_data_sec()
2733 shdr->sh_entsize = 0; in emit_elf_data_sec()
2735 data->d_type = ELF_T_BYTE; in emit_elf_data_sec()
2736 data->d_size = raw_sz; in emit_elf_data_sec()
2737 data->d_buf = (void *)raw_data; in emit_elf_data_sec()
2738 data->d_align = align; in emit_elf_data_sec()
2739 data->d_off = 0; in emit_elf_data_sec()
2748 struct btf *btf = linker->btf; in finalize_btf()
2754 if (btf__type_cnt(linker->btf) == 1) in finalize_btf()
2757 for (i = 1; i < linker->sec_cnt; i++) { in finalize_btf()
2758 struct dst_sec *sec = &linker->secs[i]; in finalize_btf()
2760 if (!sec->has_btf) in finalize_btf()
2763 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz); in finalize_btf()
2766 sec->sec_name, id); in finalize_btf()
2770 for (j = 0; j < sec->sec_var_cnt; j++) { in finalize_btf()
2771 struct btf_var_secinfo *vi = &sec->sec_vars[j]; in finalize_btf()
2773 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size)) in finalize_btf()
2774 return -EINVAL; in finalize_btf()
2784 opts.btf_ext = linker->btf_ext; in finalize_btf()
2785 err = btf__dedup(linker->btf, &opts); in finalize_btf()
2792 link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ? in finalize_btf()
2794 btf__set_endianness(linker->btf, link_endianness); in finalize_btf()
2795 if (linker->btf_ext) in finalize_btf()
2796 btf_ext__set_endianness(linker->btf_ext, link_endianness); in finalize_btf()
2799 raw_data = btf__raw_data(linker->btf, &raw_sz); in finalize_btf()
2801 return -ENOMEM; in finalize_btf()
2810 if (linker->btf_ext) { in finalize_btf()
2811 raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz); in finalize_btf()
2813 return -ENOMEM; in finalize_btf()
2833 if (!sec_data->rec_cnt) in emit_btf_ext_data()
2836 str_off = btf__add_str(linker->btf, sec_name); in emit_btf_ext_data()
2838 return -ENOMEM; in emit_btf_ext_data()
2841 sec_info->sec_name_off = str_off; in emit_btf_ext_data()
2842 sec_info->num_info = sec_data->rec_cnt; in emit_btf_ext_data()
2845 sz = sec_data->rec_cnt * sec_data->rec_sz; in emit_btf_ext_data()
2846 memcpy(cur, sec_data->recs, sz); in emit_btf_ext_data()
2849 return cur - output; in emit_btf_ext_data()
2864 for (i = 1; i < linker->sec_cnt; i++) { in finalize_btf_ext()
2865 struct dst_sec *sec = &linker->secs[i]; in finalize_btf_ext()
2867 if (sec->func_info.rec_cnt) { in finalize_btf_ext()
2869 func_rec_sz = sec->func_info.rec_sz; in finalize_btf_ext()
2870 if (func_rec_sz != sec->func_info.rec_sz) { in finalize_btf_ext()
2872 func_rec_sz, sec->func_info.rec_sz); in finalize_btf_ext()
2873 return -EINVAL; in finalize_btf_ext()
2876 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt; in finalize_btf_ext()
2878 if (sec->line_info.rec_cnt) { in finalize_btf_ext()
2880 line_rec_sz = sec->line_info.rec_sz; in finalize_btf_ext()
2881 if (line_rec_sz != sec->line_info.rec_sz) { in finalize_btf_ext()
2883 line_rec_sz, sec->line_info.rec_sz); in finalize_btf_ext()
2884 return -EINVAL; in finalize_btf_ext()
2887 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt; in finalize_btf_ext()
2889 if (sec->core_relo_info.rec_cnt) { in finalize_btf_ext()
2891 core_relo_rec_sz = sec->core_relo_info.rec_sz; in finalize_btf_ext()
2892 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) { in finalize_btf_ext()
2894 core_relo_rec_sz, sec->core_relo_info.rec_sz); in finalize_btf_ext()
2895 return -EINVAL; in finalize_btf_ext()
2898 … core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt; in finalize_btf_ext()
2921 return -ENOMEM; in finalize_btf_ext()
2924 hdr->magic = BTF_MAGIC; in finalize_btf_ext()
2925 hdr->version = BTF_VERSION; in finalize_btf_ext()
2926 hdr->flags = 0; in finalize_btf_ext()
2927 hdr->hdr_len = sizeof(struct btf_ext_header); in finalize_btf_ext()
2931 hdr->func_info_off = 0; in finalize_btf_ext()
2932 hdr->func_info_len = funcs_sz; in finalize_btf_ext()
2933 hdr->line_info_off = funcs_sz; in finalize_btf_ext()
2934 hdr->line_info_len = lines_sz; in finalize_btf_ext()
2935 hdr->core_relo_off = funcs_sz + lines_sz; in finalize_btf_ext()
2936 hdr->core_relo_len = core_relos_sz; in finalize_btf_ext()
2942 for (i = 1; i < linker->sec_cnt; i++) { in finalize_btf_ext()
2943 struct dst_sec *sec = &linker->secs[i]; in finalize_btf_ext()
2945 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info); in finalize_btf_ext()
2959 for (i = 1; i < linker->sec_cnt; i++) { in finalize_btf_ext()
2960 struct dst_sec *sec = &linker->secs[i]; in finalize_btf_ext()
2962 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info); in finalize_btf_ext()
2976 for (i = 1; i < linker->sec_cnt; i++) { in finalize_btf_ext()
2977 struct dst_sec *sec = &linker->secs[i]; in finalize_btf_ext()
2979 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info); in finalize_btf_ext()
2989 linker->btf_ext = btf_ext__new(data, total_sz); in finalize_btf_ext()
2990 err = libbpf_get_error(linker->btf_ext); in finalize_btf_ext()
2992 linker->btf_ext = NULL; in finalize_btf_ext()