166 static int init_output_elf(struct bpf_linker *linker);
168 static int bpf_linker_add_file(struct bpf_linker *linker, int fd,
171 static int linker_load_obj_file(struct bpf_linker *linker,
179 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
180 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
181 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
183 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
184 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
185 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
187 static int finalize_btf(struct bpf_linker *linker);
188 static int finalize_btf_ext(struct bpf_linker *linker);
190 void bpf_linker__free(struct bpf_linker *linker)  in bpf_linker__free()  argument
194 	if (!linker)  in bpf_linker__free()
197 	free(linker->filename);  in bpf_linker__free()
199 	if (linker->elf)  in bpf_linker__free()
200 		elf_end(linker->elf);  in bpf_linker__free()
202 	if (linker->fd >= 0 && linker->fd_is_owned)  in bpf_linker__free()
203 		close(linker->fd);  in bpf_linker__free()
205 	strset__free(linker->strtab_strs);  in bpf_linker__free()
207 	btf__free(linker->btf);  in bpf_linker__free()
208 	btf_ext__free(linker->btf_ext);  in bpf_linker__free()
210 	for (i = 1; i < linker->sec_cnt; i++) {  in bpf_linker__free()
211 		struct dst_sec *sec = &linker->secs[i];  in bpf_linker__free()
221 	free(linker->secs);  in bpf_linker__free()
223 	free(linker->glob_syms);  in bpf_linker__free()
224 	free(linker);  in bpf_linker__free()
229 	struct bpf_linker *linker;  in bpf_linker__new()  local
240 	linker = calloc(1, sizeof(*linker));  in bpf_linker__new()
241 	if (!linker)  in bpf_linker__new()
244 	linker->filename = strdup(filename);  in bpf_linker__new()
245 	if (!linker->filename) {  in bpf_linker__new()
250 	linker->fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);  in bpf_linker__new()
251 	if (linker->fd < 0) {  in bpf_linker__new()
256 	linker->fd_is_owned = true;  in bpf_linker__new()
258 	err = init_output_elf(linker);  in bpf_linker__new()
262 	return linker;  in bpf_linker__new()
265 	bpf_linker__free(linker);  in bpf_linker__new()
271 	struct bpf_linker *linker;  in bpf_linker__new_fd()  local
286 	linker = calloc(1, sizeof(*linker));  in bpf_linker__new_fd()
287 	if (!linker)  in bpf_linker__new_fd()
291 	linker->filename = strdup(filename);  in bpf_linker__new_fd()
292 	if (!linker->filename) {  in bpf_linker__new_fd()
297 	linker->fd = fd;  in bpf_linker__new_fd()
298 	linker->fd_is_owned = false;  in bpf_linker__new_fd()
300 	err = init_output_elf(linker);  in bpf_linker__new_fd()
304 	return linker;  in bpf_linker__new_fd()
307 	bpf_linker__free(linker);  in bpf_linker__new_fd()
311 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)  in add_dst_sec()  argument
313 	struct dst_sec *secs = linker->secs, *sec;  in add_dst_sec()
314 	size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;  in add_dst_sec()
321 	memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));  in add_dst_sec()
323 	linker->secs = secs;  in add_dst_sec()
324 	linker->sec_cnt = new_cnt;  in add_dst_sec()
326 	sec = &linker->secs[new_cnt - 1];  in add_dst_sec()
335 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)  in add_new_sym()  argument
337 	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];  in add_new_sym()
359 static int init_output_elf(struct bpf_linker *linker)  in init_output_elf()  argument
365 	linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);  in init_output_elf()
366 	if (!linker->elf) {  in init_output_elf()
372 	linker->elf_hdr = elf64_newehdr(linker->elf);  in init_output_elf()
373 	if (!linker->elf_hdr) {  in init_output_elf()
378 	linker->elf_hdr->e_machine = EM_BPF;  in init_output_elf()
379 	linker->elf_hdr->e_type = ET_REL;  in init_output_elf()
381 	linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE;  in init_output_elf()
385 	linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));  in init_output_elf()
386 	if (libbpf_get_error(linker->strtab_strs))  in init_output_elf()
387 		return libbpf_get_error(linker->strtab_strs);  in init_output_elf()
389 	sec = add_dst_sec(linker, ".strtab");  in init_output_elf()
393 	sec->scn = elf_newscn(linker->elf);  in init_output_elf()
409 	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);  in init_output_elf()
414 	linker->elf_hdr->e_shstrndx = sec->sec_idx;  in init_output_elf()
415 	linker->strtab_sec_idx = sec->sec_idx;  in init_output_elf()
428 	sec = add_dst_sec(linker, ".symtab");  in init_output_elf()
432 	sec->scn = elf_newscn(linker->elf);  in init_output_elf()
450 	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);  in init_output_elf()
455 	linker->symtab_sec_idx = sec->sec_idx;  in init_output_elf()
461 	sec->shdr->sh_link = linker->strtab_sec_idx;  in init_output_elf()
470 	linker->btf = btf__new_empty();  in init_output_elf()
471 	err = libbpf_get_error(linker->btf);  in init_output_elf()
476 	init_sym = add_new_sym(linker, NULL);  in init_output_elf()
490 static int bpf_linker_add_file(struct bpf_linker *linker, int fd,  in bpf_linker_add_file()  argument
499 	err = err ?: linker_load_obj_file(linker, &obj);  in bpf_linker_add_file()
500 	err = err ?: linker_append_sec_data(linker, &obj);  in bpf_linker_add_file()
501 	err = err ?: linker_append_elf_syms(linker, &obj);  in bpf_linker_add_file()
502 	err = err ?: linker_append_elf_relos(linker, &obj);  in bpf_linker_add_file()
503 	err = err ?: linker_append_btf(linker, &obj);  in bpf_linker_add_file()
504 	err = err ?: linker_append_btf_ext(linker, &obj);  in bpf_linker_add_file()
518 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,  in bpf_linker__add_file()  argument
526 	if (!linker->elf)  in bpf_linker__add_file()
536 	err = bpf_linker_add_file(linker, fd, filename);  in bpf_linker__add_file()
541 int bpf_linker__add_fd(struct bpf_linker *linker, int fd,  in bpf_linker__add_fd()  argument
550 	if (!linker->elf)  in bpf_linker__add_fd()
557 	err = bpf_linker_add_file(linker, fd, filename);  in bpf_linker__add_fd()
561 int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz,  in bpf_linker__add_buf()  argument
570 	if (!linker->elf)  in bpf_linker__add_buf()
593 	ret = bpf_linker_add_file(linker, fd, filename);  in bpf_linker__add_buf()
664 static int linker_load_obj_file(struct bpf_linker *linker,  in linker_load_obj_file()  argument
674 	unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA];  in linker_load_obj_file()
706 		linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder;  in linker_load_obj_file()
707 		linker->swapped_endian = obj_byteorder != host_byteorder;  in linker_load_obj_file()
1140 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)  in init_sec()  argument
1155 	scn = elf_newscn(linker->elf);  in init_sec()
1170 	name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);  in init_sec()
1196 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)  in find_dst_sec_by_name()  argument
1201 	for (i = 1; i < linker->sec_cnt; i++) {  in find_dst_sec_by_name()
1202 		sec = &linker->secs[i];  in find_dst_sec_by_name()
1259 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)  in extend_sec()  argument
1279 		err = init_sec(linker, dst, src);  in extend_sec()
1320 		if (linker->swapped_endian && is_exec_sec(dst))  in extend_sec()
1353 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)  in linker_append_sec_data()  argument
1365 		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);  in linker_append_sec_data()
1367 			dst_sec = add_dst_sec(linker, src_sec->sec_name);  in linker_append_sec_data()
1370 			err = init_sec(linker, dst_sec, src_sec);  in linker_append_sec_data()
1397 		err = extend_sec(linker, dst_sec, src_sec);  in linker_append_sec_data()
1405 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)  in linker_append_elf_syms()  argument
1430 		err = linker_append_elf_sym(linker, obj, sym, sym_name, i);  in linker_append_elf_syms()
1438 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)  in get_sym_by_idx()  argument
1440 	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];  in get_sym_by_idx()
1446 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)  in find_glob_sym()  argument
1452 	for (i = 0; i < linker->glob_sym_cnt; i++) {  in find_glob_sym()
1453 		glob_sym = &linker->glob_syms[i];  in find_glob_sym()
1454 		name = strset__data(linker->strtab_strs) + glob_sym->name_off;  in find_glob_sym()
1463 static struct glob_sym *add_glob_sym(struct bpf_linker *linker)  in add_glob_sym()  argument
1467 	syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,  in add_glob_sym()
1468 				   sizeof(*linker->glob_syms));  in add_glob_sym()
1472 	sym = &syms[linker->glob_sym_cnt];  in add_glob_sym()
1476 	linker->glob_syms = syms;  in add_glob_sym()
1477 	linker->glob_sym_cnt++;  in add_glob_sym()
1765 				struct bpf_linker *linker, struct glob_sym *glob_sym,  in glob_map_defs_match()  argument
1787 	t = btf__type_by_id(linker->btf, glob_sym->btf_id);  in glob_map_defs_match()
1788 	t = skip_mods_and_typedefs(linker->btf, t->type, NULL);  in glob_map_defs_match()
1789 	err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);  in glob_map_defs_match()
1800 	return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,  in glob_map_defs_match()
1805 			    struct bpf_linker *linker, struct glob_sym *glob_sym,  in glob_syms_match()  argument
1826 	if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)  in glob_syms_match()
1827 		return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);  in glob_syms_match()
1830 				  linker->btf, glob_sym->btf_id, obj->btf, btf_id))  in glob_syms_match()
1995 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,  in linker_append_elf_sym()  argument
2021 		dst_sec = &linker->secs[src_sec->dst_id];  in linker_append_elf_sym()
2060 			dst_sec = &linker->secs[src_sec->dst_id];  in linker_append_elf_sym()
2064 	glob_sym = find_glob_sym(linker, sym_name);  in linker_append_elf_sym()
2083 		if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))  in linker_append_elf_sym()
2086 		dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);  in linker_append_elf_sym()
2139 		if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,  in linker_append_elf_sym()
2151 	name_off = strset__add_str(linker->strtab_strs, sym_name);  in linker_append_elf_sym()
2155 	dst_sym = add_new_sym(linker, &dst_sym_idx);  in linker_append_elf_sym()
2174 		glob_sym = add_glob_sym(linker);  in linker_append_elf_sym()
2196 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)  in linker_append_elf_relos()  argument
2216 		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);  in linker_append_elf_relos()
2218 			dst_sec = add_dst_sec(linker, src_sec->sec_name);  in linker_append_elf_relos()
2221 			err = init_sec(linker, dst_sec, src_sec);  in linker_append_elf_relos()
2232 		dst_sec->shdr->sh_link = linker->symtab_sec_idx;  in linker_append_elf_relos()
2235 		dst_linked_sec = &linker->secs[src_linked_sec->dst_id];  in linker_append_elf_relos()
2239 		err = extend_sec(linker, dst_sec, src_sec);  in linker_append_elf_relos()
2412 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)  in linker_append_btf()  argument
2421 	start_id = btf__type_cnt(linker->btf);  in linker_append_btf()
2440 			glob_sym = find_glob_sym(linker, name);  in linker_append_btf()
2467 		id = btf__add_type(linker->btf, obj->btf, t);  in linker_append_btf()
2483 	n = btf__type_cnt(linker->btf);  in linker_append_btf()
2485 		struct btf_type *dst_t = btf_type_by_id(linker->btf, i);  in linker_append_btf()
2510 	for (i = 0; i < linker->glob_sym_cnt; i++) {  in linker_append_btf()
2511 		struct glob_sym *glob_sym = &linker->glob_syms[i];  in linker_append_btf()
2519 		glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);  in linker_append_btf()
2533 		dst_sec = &linker->secs[src_sec->dst_id];  in linker_append_btf()
2553 			t = btf_type_by_id(linker->btf, new_id);  in linker_append_btf()
2555 				name = btf__str_by_offset(linker->btf, t->name_off);  in linker_append_btf()
2556 				glob_sym = find_glob_sym(linker, name);  in linker_append_btf()
2581 				sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);  in linker_append_btf()
2630 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)  in linker_append_btf_ext()  argument
2651 		dst_sec = &linker->secs[src_sec->dst_id];  in linker_append_btf_ext()
2680 		dst_sec = &linker->secs[src_sec->dst_id];  in linker_append_btf_ext()
2697 			str_off = btf__add_str(linker->btf, s);  in linker_append_btf_ext()
2703 			str_off = btf__add_str(linker->btf, s);  in linker_append_btf_ext()
2722 		dst_sec = &linker->secs[src_sec->dst_id];  in linker_append_btf_ext()
2740 			str_off = btf__add_str(linker->btf, s);  in linker_append_btf_ext()
2752 int bpf_linker__finalize(struct bpf_linker *linker)  in bpf_linker__finalize()  argument
2759 	if (!linker->elf)  in bpf_linker__finalize()
2762 	err = finalize_btf(linker);  in bpf_linker__finalize()
2767 	strs_sz = strset__data_size(linker->strtab_strs);  in bpf_linker__finalize()
2768 	strs = strset__data(linker->strtab_strs);  in bpf_linker__finalize()
2770 	sec = &linker->secs[linker->strtab_sec_idx];  in bpf_linker__finalize()
2778 	for (i = 1; i < linker->sec_cnt; i++) {  in bpf_linker__finalize()
2779 		sec = &linker->secs[i];  in bpf_linker__finalize()
2782 		if (sec->sec_idx == linker->strtab_sec_idx)  in bpf_linker__finalize()
2790 		if (linker->swapped_endian && is_exec_sec(sec))  in bpf_linker__finalize()
2797 	if (elf_update(linker->elf, ELF_C_NULL) < 0) {  in bpf_linker__finalize()
2804 	if (elf_update(linker->elf, ELF_C_WRITE) < 0) {  in bpf_linker__finalize()
2810 	elf_end(linker->elf);  in bpf_linker__finalize()
2811 	linker->elf = NULL;  in bpf_linker__finalize()
2813 	if (linker->fd_is_owned)  in bpf_linker__finalize()
2814 		close(linker->fd);  in bpf_linker__finalize()
2815 	linker->fd = -1;  in bpf_linker__finalize()
2820 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,  in emit_elf_data_sec()  argument
2828 	name_off = strset__add_str(linker->strtab_strs, sec_name);  in emit_elf_data_sec()
2832 	scn = elf_newscn(linker->elf);  in emit_elf_data_sec()
2860 static int finalize_btf(struct bpf_linker *linker)  in finalize_btf()  argument
2864 	struct btf *btf = linker->btf;  in finalize_btf()
2870 	if (btf__type_cnt(linker->btf) == 1)  in finalize_btf()
2873 	for (i = 1; i < linker->sec_cnt; i++) {  in finalize_btf()
2874 		struct dst_sec *sec = &linker->secs[i];  in finalize_btf()
2894 	err = finalize_btf_ext(linker);  in finalize_btf()
2900 	opts.btf_ext = linker->btf_ext;  in finalize_btf()
2901 	err = btf__dedup(linker->btf, &opts);  in finalize_btf()
2908 	link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ?  in finalize_btf()
2910 	btf__set_endianness(linker->btf, link_endianness);  in finalize_btf()
2911 	if (linker->btf_ext)  in finalize_btf()
2912 		btf_ext__set_endianness(linker->btf_ext, link_endianness);  in finalize_btf()
2915 	raw_data = btf__raw_data(linker->btf, &raw_sz);  in finalize_btf()
2919 	err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);  in finalize_btf()
2926 	if (linker->btf_ext) {  in finalize_btf()
2927 		raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);  in finalize_btf()
2931 		err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);  in finalize_btf()
2941 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,  in emit_btf_ext_data()  argument
2952 	str_off = btf__add_str(linker->btf, sec_name);  in emit_btf_ext_data()
2968 static int finalize_btf_ext(struct bpf_linker *linker)  in finalize_btf_ext()  argument
2980 	for (i = 1; i < linker->sec_cnt; i++) {  in finalize_btf_ext()
2981 		struct dst_sec *sec = &linker->secs[i];  in finalize_btf_ext()
3058 		for (i = 1; i < linker->sec_cnt; i++) {  in finalize_btf_ext()
3059 			struct dst_sec *sec = &linker->secs[i];  in finalize_btf_ext()
3061 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);  in finalize_btf_ext()
3075 		for (i = 1; i < linker->sec_cnt; i++) {  in finalize_btf_ext()
3076 			struct dst_sec *sec = &linker->secs[i];  in finalize_btf_ext()
3078 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);  in finalize_btf_ext()
3092 		for (i = 1; i < linker->sec_cnt; i++) {  in finalize_btf_ext()
3093 			struct dst_sec *sec = &linker->secs[i];  in finalize_btf_ext()
3095 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);  in finalize_btf_ext()
3105 	linker->btf_ext = btf_ext__new(data, total_sz);  in finalize_btf_ext()
3106 	err = libbpf_get_error(linker->btf_ext);  in finalize_btf_ext()
3108 		linker->btf_ext = NULL;  in finalize_btf_ext()