xref: /linux/tools/lib/bpf/linker.c (revision 3fd6c59042dbba50391e30862beac979491145fe)
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2 /*
3  * BPF static linker
4  *
5  * Copyright (c) 2021 Facebook
6  */
7 #include <stdbool.h>
8 #include <stddef.h>
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <unistd.h>
13 #include <errno.h>
14 #include <linux/err.h>
15 #include <linux/btf.h>
16 #include <elf.h>
17 #include <libelf.h>
18 #include <fcntl.h>
19 #include "libbpf.h"
20 #include "btf.h"
21 #include "libbpf_internal.h"
22 #include "strset.h"
23 #include "str_error.h"
24 
25 #define BTF_EXTERN_SEC ".extern"
26 
27 struct src_sec {
28 	const char *sec_name;
29 	/* positional (not necessarily ELF) index in an array of sections */
30 	int id;
31 	/* positional (not necessarily ELF) index of a matching section in a final object file */
32 	int dst_id;
33 	/* section data offset in a matching output section */
34 	int dst_off;
35 	/* whether section is omitted from the final ELF file */
36 	bool skipped;
37 	/* whether section is an ephemeral section, not mapped to an ELF section */
38 	bool ephemeral;
39 
40 	/* ELF info */
41 	size_t sec_idx;
42 	Elf_Scn *scn;
43 	Elf64_Shdr *shdr;
44 	Elf_Data *data;
45 
46 	/* corresponding BTF DATASEC type ID */
47 	int sec_type_id;
48 };
49 
50 struct src_obj {
51 	const char *filename;
52 	int fd;
53 	Elf *elf;
54 	/* Section header strings section index */
55 	size_t shstrs_sec_idx;
56 	/* SYMTAB section index */
57 	size_t symtab_sec_idx;
58 
59 	struct btf *btf;
60 	struct btf_ext *btf_ext;
61 
62 	/* List of sections (including ephemeral). Slot zero is unused. */
63 	struct src_sec *secs;
64 	int sec_cnt;
65 
66 	/* mapping of symbol indices from src to dst ELF */
67 	int *sym_map;
68 	/* mapping from the src BTF type IDs to dst ones */
69 	int *btf_type_map;
70 };
71 
72 /* single .BTF.ext data section */
73 struct btf_ext_sec_data {
74 	size_t rec_cnt;
75 	__u32 rec_sz;
76 	void *recs;
77 };
78 
79 struct glob_sym {
80 	/* ELF symbol index */
81 	int sym_idx;
82 	/* associated section id for .ksyms, .kconfig, etc, but not .extern */
83 	int sec_id;
84 	/* extern name offset in STRTAB */
85 	int name_off;
86 	/* optional associated BTF type ID */
87 	int btf_id;
88 	/* BTF type ID to which VAR/FUNC type is pointing to; used for
89 	 * rewriting types when extern VAR/FUNC is resolved to a concrete
90 	 * definition
91 	 */
92 	int underlying_btf_id;
93 	/* sec_var index in the corresponding dst_sec, if exists */
94 	int var_idx;
95 
96 	/* extern or resolved/global symbol */
97 	bool is_extern;
98 	/* weak or strong symbol, never goes back from strong to weak */
99 	bool is_weak;
100 };
101 
102 struct dst_sec {
103 	char *sec_name;
104 	/* positional (not necessarily ELF) index in an array of sections */
105 	int id;
106 
107 	bool ephemeral;
108 
109 	/* ELF info */
110 	size_t sec_idx;
111 	Elf_Scn *scn;
112 	Elf64_Shdr *shdr;
113 	Elf_Data *data;
114 
115 	/* final output section size */
116 	int sec_sz;
117 	/* final output contents of the section */
118 	void *raw_data;
119 
120 	/* corresponding STT_SECTION symbol index in SYMTAB */
121 	int sec_sym_idx;
122 
123 	/* section's DATASEC variable info, emitted on BTF finalization */
124 	bool has_btf;
125 	int sec_var_cnt;
126 	struct btf_var_secinfo *sec_vars;
127 
128 	/* section's .BTF.ext data */
129 	struct btf_ext_sec_data func_info;
130 	struct btf_ext_sec_data line_info;
131 	struct btf_ext_sec_data core_relo_info;
132 };
133 
134 struct bpf_linker {
135 	char *filename;
136 	int fd;
137 	Elf *elf;
138 	Elf64_Ehdr *elf_hdr;
139 	bool swapped_endian;
140 
141 	/* Output sections metadata */
142 	struct dst_sec *secs;
143 	int sec_cnt;
144 
145 	struct strset *strtab_strs; /* STRTAB unique strings */
146 	size_t strtab_sec_idx; /* STRTAB section index */
147 	size_t symtab_sec_idx; /* SYMTAB section index */
148 
149 	struct btf *btf;
150 	struct btf_ext *btf_ext;
151 
152 	/* global (including extern) ELF symbols */
153 	int glob_sym_cnt;
154 	struct glob_sym *glob_syms;
155 };
156 
157 #define pr_warn_elf(fmt, ...)									\
158 	libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
159 
160 static int init_output_elf(struct bpf_linker *linker, const char *file);
161 
162 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
163 				const struct bpf_linker_file_opts *opts,
164 				struct src_obj *obj);
165 static int linker_sanity_check_elf(struct src_obj *obj);
166 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
167 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
168 static int linker_sanity_check_btf(struct src_obj *obj);
169 static int linker_sanity_check_btf_ext(struct src_obj *obj);
170 static int linker_fixup_btf(struct src_obj *obj);
171 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
172 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
173 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
174 				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
175 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
176 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
177 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
178 
179 static int finalize_btf(struct bpf_linker *linker);
180 static int finalize_btf_ext(struct bpf_linker *linker);
181 
bpf_linker__free(struct bpf_linker * linker)182 void bpf_linker__free(struct bpf_linker *linker)
183 {
184 	int i;
185 
186 	if (!linker)
187 		return;
188 
189 	free(linker->filename);
190 
191 	if (linker->elf)
192 		elf_end(linker->elf);
193 
194 	if (linker->fd >= 0)
195 		close(linker->fd);
196 
197 	strset__free(linker->strtab_strs);
198 
199 	btf__free(linker->btf);
200 	btf_ext__free(linker->btf_ext);
201 
202 	for (i = 1; i < linker->sec_cnt; i++) {
203 		struct dst_sec *sec = &linker->secs[i];
204 
205 		free(sec->sec_name);
206 		free(sec->raw_data);
207 		free(sec->sec_vars);
208 
209 		free(sec->func_info.recs);
210 		free(sec->line_info.recs);
211 		free(sec->core_relo_info.recs);
212 	}
213 	free(linker->secs);
214 
215 	free(linker->glob_syms);
216 	free(linker);
217 }
218 
bpf_linker__new(const char * filename,struct bpf_linker_opts * opts)219 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
220 {
221 	struct bpf_linker *linker;
222 	int err;
223 
224 	if (!OPTS_VALID(opts, bpf_linker_opts))
225 		return errno = EINVAL, NULL;
226 
227 	if (elf_version(EV_CURRENT) == EV_NONE) {
228 		pr_warn_elf("libelf initialization failed");
229 		return errno = EINVAL, NULL;
230 	}
231 
232 	linker = calloc(1, sizeof(*linker));
233 	if (!linker)
234 		return errno = ENOMEM, NULL;
235 
236 	linker->fd = -1;
237 
238 	err = init_output_elf(linker, filename);
239 	if (err)
240 		goto err_out;
241 
242 	return linker;
243 
244 err_out:
245 	bpf_linker__free(linker);
246 	return errno = -err, NULL;
247 }
248 
add_dst_sec(struct bpf_linker * linker,const char * sec_name)249 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
250 {
251 	struct dst_sec *secs = linker->secs, *sec;
252 	size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
253 
254 	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
255 	if (!secs)
256 		return NULL;
257 
258 	/* zero out newly allocated memory */
259 	memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
260 
261 	linker->secs = secs;
262 	linker->sec_cnt = new_cnt;
263 
264 	sec = &linker->secs[new_cnt - 1];
265 	sec->id = new_cnt - 1;
266 	sec->sec_name = strdup(sec_name);
267 	if (!sec->sec_name)
268 		return NULL;
269 
270 	return sec;
271 }
272 
add_new_sym(struct bpf_linker * linker,size_t * sym_idx)273 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
274 {
275 	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
276 	Elf64_Sym *syms, *sym;
277 	size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
278 
279 	syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
280 	if (!syms)
281 		return NULL;
282 
283 	sym = &syms[sym_cnt];
284 	memset(sym, 0, sizeof(*sym));
285 
286 	symtab->raw_data = syms;
287 	symtab->sec_sz += sizeof(*sym);
288 	symtab->shdr->sh_size += sizeof(*sym);
289 	symtab->data->d_size += sizeof(*sym);
290 
291 	if (sym_idx)
292 		*sym_idx = sym_cnt;
293 
294 	return sym;
295 }
296 
init_output_elf(struct bpf_linker * linker,const char * file)297 static int init_output_elf(struct bpf_linker *linker, const char *file)
298 {
299 	int err, str_off;
300 	Elf64_Sym *init_sym;
301 	struct dst_sec *sec;
302 
303 	linker->filename = strdup(file);
304 	if (!linker->filename)
305 		return -ENOMEM;
306 
307 	linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
308 	if (linker->fd < 0) {
309 		err = -errno;
310 		pr_warn("failed to create '%s': %s\n", file, errstr(err));
311 		return err;
312 	}
313 
314 	linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
315 	if (!linker->elf) {
316 		pr_warn_elf("failed to create ELF object");
317 		return -EINVAL;
318 	}
319 
320 	/* ELF header */
321 	linker->elf_hdr = elf64_newehdr(linker->elf);
322 	if (!linker->elf_hdr) {
323 		pr_warn_elf("failed to create ELF header");
324 		return -EINVAL;
325 	}
326 
327 	linker->elf_hdr->e_machine = EM_BPF;
328 	linker->elf_hdr->e_type = ET_REL;
329 	/* Set unknown ELF endianness, assign later from input files */
330 	linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE;
331 
332 	/* STRTAB */
333 	/* initialize strset with an empty string to conform to ELF */
334 	linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
335 	if (libbpf_get_error(linker->strtab_strs))
336 		return libbpf_get_error(linker->strtab_strs);
337 
338 	sec = add_dst_sec(linker, ".strtab");
339 	if (!sec)
340 		return -ENOMEM;
341 
342 	sec->scn = elf_newscn(linker->elf);
343 	if (!sec->scn) {
344 		pr_warn_elf("failed to create STRTAB section");
345 		return -EINVAL;
346 	}
347 
348 	sec->shdr = elf64_getshdr(sec->scn);
349 	if (!sec->shdr)
350 		return -EINVAL;
351 
352 	sec->data = elf_newdata(sec->scn);
353 	if (!sec->data) {
354 		pr_warn_elf("failed to create STRTAB data");
355 		return -EINVAL;
356 	}
357 
358 	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
359 	if (str_off < 0)
360 		return str_off;
361 
362 	sec->sec_idx = elf_ndxscn(sec->scn);
363 	linker->elf_hdr->e_shstrndx = sec->sec_idx;
364 	linker->strtab_sec_idx = sec->sec_idx;
365 
366 	sec->shdr->sh_name = str_off;
367 	sec->shdr->sh_type = SHT_STRTAB;
368 	sec->shdr->sh_flags = SHF_STRINGS;
369 	sec->shdr->sh_offset = 0;
370 	sec->shdr->sh_link = 0;
371 	sec->shdr->sh_info = 0;
372 	sec->shdr->sh_addralign = 1;
373 	sec->shdr->sh_size = sec->sec_sz = 0;
374 	sec->shdr->sh_entsize = 0;
375 
376 	/* SYMTAB */
377 	sec = add_dst_sec(linker, ".symtab");
378 	if (!sec)
379 		return -ENOMEM;
380 
381 	sec->scn = elf_newscn(linker->elf);
382 	if (!sec->scn) {
383 		pr_warn_elf("failed to create SYMTAB section");
384 		return -EINVAL;
385 	}
386 
387 	sec->shdr = elf64_getshdr(sec->scn);
388 	if (!sec->shdr)
389 		return -EINVAL;
390 
391 	sec->data = elf_newdata(sec->scn);
392 	if (!sec->data) {
393 		pr_warn_elf("failed to create SYMTAB data");
394 		return -EINVAL;
395 	}
396 	/* Ensure libelf translates byte-order of symbol records */
397 	sec->data->d_type = ELF_T_SYM;
398 
399 	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
400 	if (str_off < 0)
401 		return str_off;
402 
403 	sec->sec_idx = elf_ndxscn(sec->scn);
404 	linker->symtab_sec_idx = sec->sec_idx;
405 
406 	sec->shdr->sh_name = str_off;
407 	sec->shdr->sh_type = SHT_SYMTAB;
408 	sec->shdr->sh_flags = 0;
409 	sec->shdr->sh_offset = 0;
410 	sec->shdr->sh_link = linker->strtab_sec_idx;
411 	/* sh_info should be one greater than the index of the last local
412 	 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
413 	 */
414 	sec->shdr->sh_info = 0;
415 	sec->shdr->sh_addralign = 8;
416 	sec->shdr->sh_entsize = sizeof(Elf64_Sym);
417 
418 	/* .BTF */
419 	linker->btf = btf__new_empty();
420 	err = libbpf_get_error(linker->btf);
421 	if (err)
422 		return err;
423 
424 	/* add the special all-zero symbol */
425 	init_sym = add_new_sym(linker, NULL);
426 	if (!init_sym)
427 		return -EINVAL;
428 
429 	init_sym->st_name = 0;
430 	init_sym->st_info = 0;
431 	init_sym->st_other = 0;
432 	init_sym->st_shndx = SHN_UNDEF;
433 	init_sym->st_value = 0;
434 	init_sym->st_size = 0;
435 
436 	return 0;
437 }
438 
bpf_linker__add_file(struct bpf_linker * linker,const char * filename,const struct bpf_linker_file_opts * opts)439 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
440 			 const struct bpf_linker_file_opts *opts)
441 {
442 	struct src_obj obj = {};
443 	int err = 0;
444 
445 	if (!OPTS_VALID(opts, bpf_linker_file_opts))
446 		return libbpf_err(-EINVAL);
447 
448 	if (!linker->elf)
449 		return libbpf_err(-EINVAL);
450 
451 	err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
452 	err = err ?: linker_append_sec_data(linker, &obj);
453 	err = err ?: linker_append_elf_syms(linker, &obj);
454 	err = err ?: linker_append_elf_relos(linker, &obj);
455 	err = err ?: linker_append_btf(linker, &obj);
456 	err = err ?: linker_append_btf_ext(linker, &obj);
457 
458 	/* free up src_obj resources */
459 	free(obj.btf_type_map);
460 	btf__free(obj.btf);
461 	btf_ext__free(obj.btf_ext);
462 	free(obj.secs);
463 	free(obj.sym_map);
464 	if (obj.elf)
465 		elf_end(obj.elf);
466 	if (obj.fd >= 0)
467 		close(obj.fd);
468 
469 	return libbpf_err(err);
470 }
471 
is_dwarf_sec_name(const char * name)472 static bool is_dwarf_sec_name(const char *name)
473 {
474 	/* approximation, but the actual list is too long */
475 	return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
476 }
477 
is_ignored_sec(struct src_sec * sec)478 static bool is_ignored_sec(struct src_sec *sec)
479 {
480 	Elf64_Shdr *shdr = sec->shdr;
481 	const char *name = sec->sec_name;
482 
483 	/* no special handling of .strtab */
484 	if (shdr->sh_type == SHT_STRTAB)
485 		return true;
486 
487 	/* ignore .llvm_addrsig section as well */
488 	if (shdr->sh_type == SHT_LLVM_ADDRSIG)
489 		return true;
490 
491 	/* no subprograms will lead to an empty .text section, ignore it */
492 	if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
493 	    strcmp(sec->sec_name, ".text") == 0)
494 		return true;
495 
496 	/* DWARF sections */
497 	if (is_dwarf_sec_name(sec->sec_name))
498 		return true;
499 
500 	if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
501 		name += sizeof(".rel") - 1;
502 		/* DWARF section relocations */
503 		if (is_dwarf_sec_name(name))
504 			return true;
505 
506 		/* .BTF and .BTF.ext don't need relocations */
507 		if (strcmp(name, BTF_ELF_SEC) == 0 ||
508 		    strcmp(name, BTF_EXT_ELF_SEC) == 0)
509 			return true;
510 	}
511 
512 	return false;
513 }
514 
add_src_sec(struct src_obj * obj,const char * sec_name)515 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
516 {
517 	struct src_sec *secs = obj->secs, *sec;
518 	size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
519 
520 	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
521 	if (!secs)
522 		return NULL;
523 
524 	/* zero out newly allocated memory */
525 	memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
526 
527 	obj->secs = secs;
528 	obj->sec_cnt = new_cnt;
529 
530 	sec = &obj->secs[new_cnt - 1];
531 	sec->id = new_cnt - 1;
532 	sec->sec_name = sec_name;
533 
534 	return sec;
535 }
536 
linker_load_obj_file(struct bpf_linker * linker,const char * filename,const struct bpf_linker_file_opts * opts,struct src_obj * obj)537 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
538 				const struct bpf_linker_file_opts *opts,
539 				struct src_obj *obj)
540 {
541 	int err = 0;
542 	Elf_Scn *scn;
543 	Elf_Data *data;
544 	Elf64_Ehdr *ehdr;
545 	Elf64_Shdr *shdr;
546 	struct src_sec *sec;
547 	unsigned char obj_byteorder;
548 	unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA];
549 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
550 	const unsigned char host_byteorder = ELFDATA2LSB;
551 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
552 	const unsigned char host_byteorder = ELFDATA2MSB;
553 #else
554 #error "Unknown __BYTE_ORDER__"
555 #endif
556 
557 	pr_debug("linker: adding object file '%s'...\n", filename);
558 
559 	obj->filename = filename;
560 
561 	obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
562 	if (obj->fd < 0) {
563 		err = -errno;
564 		pr_warn("failed to open file '%s': %s\n", filename, errstr(err));
565 		return err;
566 	}
567 	obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
568 	if (!obj->elf) {
569 		err = -errno;
570 		pr_warn_elf("failed to parse ELF file '%s'", filename);
571 		return err;
572 	}
573 
574 	/* Sanity check ELF file high-level properties */
575 	ehdr = elf64_getehdr(obj->elf);
576 	if (!ehdr) {
577 		err = -errno;
578 		pr_warn_elf("failed to get ELF header for %s", filename);
579 		return err;
580 	}
581 
582 	/* Linker output endianness set by first input object */
583 	obj_byteorder = ehdr->e_ident[EI_DATA];
584 	if (obj_byteorder != ELFDATA2LSB && obj_byteorder != ELFDATA2MSB) {
585 		err = -EOPNOTSUPP;
586 		pr_warn("unknown byte order of ELF file %s\n", filename);
587 		return err;
588 	}
589 	if (link_byteorder == ELFDATANONE) {
590 		linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder;
591 		linker->swapped_endian = obj_byteorder != host_byteorder;
592 		pr_debug("linker: set %s-endian output byte order\n",
593 			 obj_byteorder == ELFDATA2MSB ? "big" : "little");
594 	} else if (link_byteorder != obj_byteorder) {
595 		err = -EOPNOTSUPP;
596 		pr_warn("byte order mismatch with ELF file %s\n", filename);
597 		return err;
598 	}
599 
600 	if (ehdr->e_type != ET_REL
601 	    || ehdr->e_machine != EM_BPF
602 	    || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
603 		err = -EOPNOTSUPP;
604 		pr_warn_elf("unsupported kind of ELF file %s", filename);
605 		return err;
606 	}
607 
608 	if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
609 		err = -errno;
610 		pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
611 		return err;
612 	}
613 
614 	scn = NULL;
615 	while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
616 		size_t sec_idx = elf_ndxscn(scn);
617 		const char *sec_name;
618 
619 		shdr = elf64_getshdr(scn);
620 		if (!shdr) {
621 			err = -errno;
622 			pr_warn_elf("failed to get section #%zu header for %s",
623 				    sec_idx, filename);
624 			return err;
625 		}
626 
627 		sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
628 		if (!sec_name) {
629 			err = -errno;
630 			pr_warn_elf("failed to get section #%zu name for %s",
631 				    sec_idx, filename);
632 			return err;
633 		}
634 
635 		data = elf_getdata(scn, 0);
636 		if (!data) {
637 			err = -errno;
638 			pr_warn_elf("failed to get section #%zu (%s) data from %s",
639 				    sec_idx, sec_name, filename);
640 			return err;
641 		}
642 
643 		sec = add_src_sec(obj, sec_name);
644 		if (!sec)
645 			return -ENOMEM;
646 
647 		sec->scn = scn;
648 		sec->shdr = shdr;
649 		sec->data = data;
650 		sec->sec_idx = elf_ndxscn(scn);
651 
652 		if (is_ignored_sec(sec)) {
653 			sec->skipped = true;
654 			continue;
655 		}
656 
657 		switch (shdr->sh_type) {
658 		case SHT_SYMTAB:
659 			if (obj->symtab_sec_idx) {
660 				err = -EOPNOTSUPP;
661 				pr_warn("multiple SYMTAB sections found, not supported\n");
662 				return err;
663 			}
664 			obj->symtab_sec_idx = sec_idx;
665 			break;
666 		case SHT_STRTAB:
667 			/* we'll construct our own string table */
668 			break;
669 		case SHT_PROGBITS:
670 			if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
671 				obj->btf = btf__new(data->d_buf, shdr->sh_size);
672 				err = libbpf_get_error(obj->btf);
673 				if (err) {
674 					pr_warn("failed to parse .BTF from %s: %s\n",
675 						filename, errstr(err));
676 					return err;
677 				}
678 				sec->skipped = true;
679 				continue;
680 			}
681 			if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
682 				obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
683 				err = libbpf_get_error(obj->btf_ext);
684 				if (err) {
685 					pr_warn("failed to parse .BTF.ext from '%s': %s\n",
686 						filename, errstr(err));
687 					return err;
688 				}
689 				sec->skipped = true;
690 				continue;
691 			}
692 
693 			/* data & code */
694 			break;
695 		case SHT_NOBITS:
696 			/* BSS */
697 			break;
698 		case SHT_REL:
699 			/* relocations */
700 			break;
701 		default:
702 			pr_warn("unrecognized section #%zu (%s) in %s\n",
703 				sec_idx, sec_name, filename);
704 			err = -EINVAL;
705 			return err;
706 		}
707 	}
708 
709 	err = err ?: linker_sanity_check_elf(obj);
710 	err = err ?: linker_sanity_check_btf(obj);
711 	err = err ?: linker_sanity_check_btf_ext(obj);
712 	err = err ?: linker_fixup_btf(obj);
713 
714 	return err;
715 }
716 
linker_sanity_check_elf(struct src_obj * obj)717 static int linker_sanity_check_elf(struct src_obj *obj)
718 {
719 	struct src_sec *sec;
720 	int i, err;
721 
722 	if (!obj->symtab_sec_idx) {
723 		pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
724 		return -EINVAL;
725 	}
726 	if (!obj->shstrs_sec_idx) {
727 		pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
728 		return -EINVAL;
729 	}
730 
731 	for (i = 1; i < obj->sec_cnt; i++) {
732 		sec = &obj->secs[i];
733 
734 		if (sec->sec_name[0] == '\0') {
735 			pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
736 			return -EINVAL;
737 		}
738 
739 		if (is_dwarf_sec_name(sec->sec_name))
740 			continue;
741 
742 		if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {
743 			pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",
744 				sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
745 				obj->filename);
746 			return -EINVAL;
747 		}
748 		if (sec->shdr->sh_addralign != sec->data->d_align) {
749 			pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",
750 				sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
751 				(long long unsigned)sec->data->d_align, obj->filename);
752 			return -EINVAL;
753 		}
754 
755 		if (sec->shdr->sh_size != sec->data->d_size) {
756 			pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",
757 				sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
758 				(long long unsigned)sec->data->d_size, obj->filename);
759 			return -EINVAL;
760 		}
761 
762 		switch (sec->shdr->sh_type) {
763 		case SHT_SYMTAB:
764 			err = linker_sanity_check_elf_symtab(obj, sec);
765 			if (err)
766 				return err;
767 			break;
768 		case SHT_STRTAB:
769 			break;
770 		case SHT_PROGBITS:
771 			if (sec->shdr->sh_flags & SHF_EXECINSTR) {
772 				if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {
773 					pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",
774 						sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
775 						obj->filename);
776 					return -EINVAL;
777 				}
778 			}
779 			break;
780 		case SHT_NOBITS:
781 			break;
782 		case SHT_REL:
783 			err = linker_sanity_check_elf_relos(obj, sec);
784 			if (err)
785 				return err;
786 			break;
787 		case SHT_LLVM_ADDRSIG:
788 			break;
789 		default:
790 			pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
791 				sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
792 			return -EINVAL;
793 		}
794 	}
795 
796 	return 0;
797 }
798 
linker_sanity_check_elf_symtab(struct src_obj * obj,struct src_sec * sec)799 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
800 {
801 	struct src_sec *link_sec;
802 	Elf64_Sym *sym;
803 	int i, n;
804 
805 	if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
806 		return -EINVAL;
807 	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
808 		return -EINVAL;
809 
810 	if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
811 		pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
812 			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
813 		return -EINVAL;
814 	}
815 	link_sec = &obj->secs[sec->shdr->sh_link];
816 	if (link_sec->shdr->sh_type != SHT_STRTAB) {
817 		pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
818 			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
819 		return -EINVAL;
820 	}
821 
822 	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
823 	sym = sec->data->d_buf;
824 	for (i = 0; i < n; i++, sym++) {
825 		int sym_type = ELF64_ST_TYPE(sym->st_info);
826 		int sym_bind = ELF64_ST_BIND(sym->st_info);
827 		int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
828 
829 		if (i == 0) {
830 			if (sym->st_name != 0 || sym->st_info != 0
831 			    || sym->st_other != 0 || sym->st_shndx != 0
832 			    || sym->st_value != 0 || sym->st_size != 0) {
833 				pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
834 				return -EINVAL;
835 			}
836 			continue;
837 		}
838 		if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
839 			pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
840 				i, sec->sec_idx, sym_bind);
841 			return -EINVAL;
842 		}
843 		if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
844 			pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
845 				i, sec->sec_idx, sym_vis);
846 			return -EINVAL;
847 		}
848 		if (sym->st_shndx == 0) {
849 			if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
850 			    || sym->st_value != 0 || sym->st_size != 0) {
851 				pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
852 					i, obj->filename);
853 
854 				return -EINVAL;
855 			}
856 			continue;
857 		}
858 		if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
859 			pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
860 				i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
861 			return -EINVAL;
862 		}
863 		if (sym_type == STT_SECTION) {
864 			if (sym->st_value != 0)
865 				return -EINVAL;
866 			continue;
867 		}
868 	}
869 
870 	return 0;
871 }
872 
linker_sanity_check_elf_relos(struct src_obj * obj,struct src_sec * sec)873 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
874 {
875 	struct src_sec *link_sec, *sym_sec;
876 	Elf64_Rel *relo;
877 	int i, n;
878 
879 	if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
880 		return -EINVAL;
881 	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
882 		return -EINVAL;
883 
884 	/* SHT_REL's sh_link should point to SYMTAB */
885 	if (sec->shdr->sh_link != obj->symtab_sec_idx) {
886 		pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
887 			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
888 		return -EINVAL;
889 	}
890 
891 	/* SHT_REL's sh_info points to relocated section */
892 	if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
893 		pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
894 			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
895 		return -EINVAL;
896 	}
897 	link_sec = &obj->secs[sec->shdr->sh_info];
898 
899 	/* .rel<secname> -> <secname> pattern is followed */
900 	if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
901 	    || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
902 		pr_warn("ELF relo section #%zu name has invalid name in %s\n",
903 			sec->sec_idx, obj->filename);
904 		return -EINVAL;
905 	}
906 
907 	/* don't further validate relocations for ignored sections */
908 	if (link_sec->skipped)
909 		return 0;
910 
911 	/* relocatable section is data or instructions */
912 	if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
913 		pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
914 			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
915 		return -EINVAL;
916 	}
917 
918 	/* check sanity of each relocation */
919 	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
920 	relo = sec->data->d_buf;
921 	sym_sec = &obj->secs[obj->symtab_sec_idx];
922 	for (i = 0; i < n; i++, relo++) {
923 		size_t sym_idx = ELF64_R_SYM(relo->r_info);
924 		size_t sym_type = ELF64_R_TYPE(relo->r_info);
925 
926 		if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
927 		    sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
928 			pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
929 				i, sec->sec_idx, sym_type, obj->filename);
930 			return -EINVAL;
931 		}
932 
933 		if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
934 			pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
935 				i, sec->sec_idx, sym_idx, obj->filename);
936 			return -EINVAL;
937 		}
938 
939 		if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
940 			if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
941 				pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
942 					i, sec->sec_idx, sym_idx, obj->filename);
943 				return -EINVAL;
944 			}
945 		}
946 	}
947 
948 	return 0;
949 }
950 
check_btf_type_id(__u32 * type_id,void * ctx)951 static int check_btf_type_id(__u32 *type_id, void *ctx)
952 {
953 	struct btf *btf = ctx;
954 
955 	if (*type_id >= btf__type_cnt(btf))
956 		return -EINVAL;
957 
958 	return 0;
959 }
960 
check_btf_str_off(__u32 * str_off,void * ctx)961 static int check_btf_str_off(__u32 *str_off, void *ctx)
962 {
963 	struct btf *btf = ctx;
964 	const char *s;
965 
966 	s = btf__str_by_offset(btf, *str_off);
967 
968 	if (!s)
969 		return -EINVAL;
970 
971 	return 0;
972 }
973 
linker_sanity_check_btf(struct src_obj * obj)974 static int linker_sanity_check_btf(struct src_obj *obj)
975 {
976 	struct btf_type *t;
977 	int i, n, err;
978 
979 	if (!obj->btf)
980 		return 0;
981 
982 	n = btf__type_cnt(obj->btf);
983 	for (i = 1; i < n; i++) {
984 		struct btf_field_iter it;
985 		__u32 *type_id, *str_off;
986 
987 		t = btf_type_by_id(obj->btf, i);
988 
989 		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
990 		if (err)
991 			return err;
992 		while ((type_id = btf_field_iter_next(&it))) {
993 			if (*type_id >= n)
994 				return -EINVAL;
995 		}
996 
997 		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
998 		if (err)
999 			return err;
1000 		while ((str_off = btf_field_iter_next(&it))) {
1001 			if (!btf__str_by_offset(obj->btf, *str_off))
1002 				return -EINVAL;
1003 		}
1004 	}
1005 
1006 	return 0;
1007 }
1008 
linker_sanity_check_btf_ext(struct src_obj * obj)1009 static int linker_sanity_check_btf_ext(struct src_obj *obj)
1010 {
1011 	int err = 0;
1012 
1013 	if (!obj->btf_ext)
1014 		return 0;
1015 
1016 	/* can't use .BTF.ext without .BTF */
1017 	if (!obj->btf)
1018 		return -EINVAL;
1019 
1020 	err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
1021 	err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
1022 	if (err)
1023 		return err;
1024 
1025 	return 0;
1026 }
1027 
init_sec(struct bpf_linker * linker,struct dst_sec * dst_sec,struct src_sec * src_sec)1028 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
1029 {
1030 	Elf_Scn *scn;
1031 	Elf_Data *data;
1032 	Elf64_Shdr *shdr;
1033 	int name_off;
1034 
1035 	dst_sec->sec_sz = 0;
1036 	dst_sec->sec_idx = 0;
1037 	dst_sec->ephemeral = src_sec->ephemeral;
1038 
1039 	/* ephemeral sections are just thin section shells lacking most parts */
1040 	if (src_sec->ephemeral)
1041 		return 0;
1042 
1043 	scn = elf_newscn(linker->elf);
1044 	if (!scn)
1045 		return -ENOMEM;
1046 	data = elf_newdata(scn);
1047 	if (!data)
1048 		return -ENOMEM;
1049 	shdr = elf64_getshdr(scn);
1050 	if (!shdr)
1051 		return -ENOMEM;
1052 
1053 	dst_sec->scn = scn;
1054 	dst_sec->shdr = shdr;
1055 	dst_sec->data = data;
1056 	dst_sec->sec_idx = elf_ndxscn(scn);
1057 
1058 	name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1059 	if (name_off < 0)
1060 		return name_off;
1061 
1062 	shdr->sh_name = name_off;
1063 	shdr->sh_type = src_sec->shdr->sh_type;
1064 	shdr->sh_flags = src_sec->shdr->sh_flags;
1065 	shdr->sh_size = 0;
1066 	/* sh_link and sh_info have different meaning for different types of
1067 	 * sections, so we leave it up to the caller code to fill them in, if
1068 	 * necessary
1069 	 */
1070 	shdr->sh_link = 0;
1071 	shdr->sh_info = 0;
1072 	shdr->sh_addralign = src_sec->shdr->sh_addralign;
1073 	shdr->sh_entsize = src_sec->shdr->sh_entsize;
1074 
1075 	data->d_type = src_sec->data->d_type;
1076 	data->d_size = 0;
1077 	data->d_buf = NULL;
1078 	data->d_align = src_sec->data->d_align;
1079 	data->d_off = 0;
1080 
1081 	return 0;
1082 }
1083 
find_dst_sec_by_name(struct bpf_linker * linker,const char * sec_name)1084 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1085 {
1086 	struct dst_sec *sec;
1087 	int i;
1088 
1089 	for (i = 1; i < linker->sec_cnt; i++) {
1090 		sec = &linker->secs[i];
1091 
1092 		if (strcmp(sec->sec_name, sec_name) == 0)
1093 			return sec;
1094 	}
1095 
1096 	return NULL;
1097 }
1098 
secs_match(struct dst_sec * dst,struct src_sec * src)1099 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1100 {
1101 	if (dst->ephemeral || src->ephemeral)
1102 		return true;
1103 
1104 	if (dst->shdr->sh_type != src->shdr->sh_type) {
1105 		pr_warn("sec %s types mismatch\n", dst->sec_name);
1106 		return false;
1107 	}
1108 	if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1109 		pr_warn("sec %s flags mismatch\n", dst->sec_name);
1110 		return false;
1111 	}
1112 	if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1113 		pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1114 		return false;
1115 	}
1116 
1117 	return true;
1118 }
1119 
sec_content_is_same(struct dst_sec * dst_sec,struct src_sec * src_sec)1120 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1121 {
1122 	if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1123 		return false;
1124 	if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1125 		return false;
1126 	return true;
1127 }
1128 
is_exec_sec(struct dst_sec * sec)1129 static bool is_exec_sec(struct dst_sec *sec)
1130 {
1131 	if (!sec || sec->ephemeral)
1132 		return false;
1133 	return (sec->shdr->sh_type == SHT_PROGBITS) &&
1134 	       (sec->shdr->sh_flags & SHF_EXECINSTR);
1135 }
1136 
exec_sec_bswap(void * raw_data,int size)1137 static void exec_sec_bswap(void *raw_data, int size)
1138 {
1139 	const int insn_cnt = size / sizeof(struct bpf_insn);
1140 	struct bpf_insn *insn = raw_data;
1141 	int i;
1142 
1143 	for (i = 0; i < insn_cnt; i++, insn++)
1144 		bpf_insn_bswap(insn);
1145 }
1146 
extend_sec(struct bpf_linker * linker,struct dst_sec * dst,struct src_sec * src)1147 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1148 {
1149 	void *tmp;
1150 	size_t dst_align, src_align;
1151 	size_t dst_align_sz, dst_final_sz;
1152 	int err;
1153 
1154 	/* Ephemeral source section doesn't contribute anything to ELF
1155 	 * section data.
1156 	 */
1157 	if (src->ephemeral)
1158 		return 0;
1159 
1160 	/* Some sections (like .maps) can contain both externs (and thus be
1161 	 * ephemeral) and non-externs (map definitions). So it's possible that
1162 	 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1163 	 * first non-ephemeral entity appears. In such case, we add ELF
1164 	 * section, data, etc.
1165 	 */
1166 	if (dst->ephemeral) {
1167 		err = init_sec(linker, dst, src);
1168 		if (err)
1169 			return err;
1170 	}
1171 
1172 	dst_align = dst->shdr->sh_addralign;
1173 	src_align = src->shdr->sh_addralign;
1174 	if (dst_align == 0)
1175 		dst_align = 1;
1176 	if (dst_align < src_align)
1177 		dst_align = src_align;
1178 
1179 	dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1180 
1181 	/* no need to re-align final size */
1182 	dst_final_sz = dst_align_sz + src->shdr->sh_size;
1183 
1184 	if (src->shdr->sh_type != SHT_NOBITS) {
1185 		tmp = realloc(dst->raw_data, dst_final_sz);
1186 		/* If dst_align_sz == 0, realloc() behaves in a special way:
1187 		 * 1. When dst->raw_data is NULL it returns:
1188 		 *    "either NULL or a pointer suitable to be passed to free()" [1].
1189 		 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1190 		 *    thus invalidating any "pointer suitable to be passed to free()" obtained
1191 		 *    at step (1).
1192 		 *
1193 		 * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1194 		 * dst->raw_data would be freed again in bpf_linker__free().
1195 		 *
1196 		 * [1] man 3 realloc
1197 		 */
1198 		if (!tmp && dst_align_sz > 0)
1199 			return -ENOMEM;
1200 		dst->raw_data = tmp;
1201 
1202 		/* pad dst section, if it's alignment forced size increase */
1203 		memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1204 		/* now copy src data at a properly aligned offset */
1205 		memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1206 
1207 		/* convert added bpf insns to native byte-order */
1208 		if (linker->swapped_endian && is_exec_sec(dst))
1209 			exec_sec_bswap(dst->raw_data + dst_align_sz, src->shdr->sh_size);
1210 	}
1211 
1212 	dst->sec_sz = dst_final_sz;
1213 	dst->shdr->sh_size = dst_final_sz;
1214 	dst->data->d_size = dst_final_sz;
1215 
1216 	dst->shdr->sh_addralign = dst_align;
1217 	dst->data->d_align = dst_align;
1218 
1219 	src->dst_off = dst_align_sz;
1220 
1221 	return 0;
1222 }
1223 
is_data_sec(struct src_sec * sec)1224 static bool is_data_sec(struct src_sec *sec)
1225 {
1226 	if (!sec || sec->skipped)
1227 		return false;
1228 	/* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1229 	if (sec->ephemeral)
1230 		return true;
1231 	return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1232 }
1233 
is_relo_sec(struct src_sec * sec)1234 static bool is_relo_sec(struct src_sec *sec)
1235 {
1236 	if (!sec || sec->skipped || sec->ephemeral)
1237 		return false;
1238 	return sec->shdr->sh_type == SHT_REL;
1239 }
1240 
linker_append_sec_data(struct bpf_linker * linker,struct src_obj * obj)1241 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1242 {
1243 	int i, err;
1244 
1245 	for (i = 1; i < obj->sec_cnt; i++) {
1246 		struct src_sec *src_sec;
1247 		struct dst_sec *dst_sec;
1248 
1249 		src_sec = &obj->secs[i];
1250 		if (!is_data_sec(src_sec))
1251 			continue;
1252 
1253 		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1254 		if (!dst_sec) {
1255 			dst_sec = add_dst_sec(linker, src_sec->sec_name);
1256 			if (!dst_sec)
1257 				return -ENOMEM;
1258 			err = init_sec(linker, dst_sec, src_sec);
1259 			if (err) {
1260 				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1261 				return err;
1262 			}
1263 		} else {
1264 			if (!secs_match(dst_sec, src_sec)) {
1265 				pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1266 				return -1;
1267 			}
1268 
1269 			/* "license" and "version" sections are deduped */
1270 			if (strcmp(src_sec->sec_name, "license") == 0
1271 			    || strcmp(src_sec->sec_name, "version") == 0) {
1272 				if (!sec_content_is_same(dst_sec, src_sec)) {
1273 					pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1274 					return -EINVAL;
1275 				}
1276 				src_sec->skipped = true;
1277 				src_sec->dst_id = dst_sec->id;
1278 				continue;
1279 			}
1280 		}
1281 
1282 		/* record mapped section index */
1283 		src_sec->dst_id = dst_sec->id;
1284 
1285 		err = extend_sec(linker, dst_sec, src_sec);
1286 		if (err)
1287 			return err;
1288 	}
1289 
1290 	return 0;
1291 }
1292 
linker_append_elf_syms(struct bpf_linker * linker,struct src_obj * obj)1293 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1294 {
1295 	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1296 	Elf64_Sym *sym = symtab->data->d_buf;
1297 	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1298 	int str_sec_idx = symtab->shdr->sh_link;
1299 	const char *sym_name;
1300 
1301 	obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1302 	if (!obj->sym_map)
1303 		return -ENOMEM;
1304 
1305 	for (i = 0; i < n; i++, sym++) {
1306 		/* We already validated all-zero symbol #0 and we already
1307 		 * appended it preventively to the final SYMTAB, so skip it.
1308 		 */
1309 		if (i == 0)
1310 			continue;
1311 
1312 		sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1313 		if (!sym_name) {
1314 			pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1315 			return -EINVAL;
1316 		}
1317 
1318 		err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1319 		if (err)
1320 			return err;
1321 	}
1322 
1323 	return 0;
1324 }
1325 
get_sym_by_idx(struct bpf_linker * linker,size_t sym_idx)1326 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1327 {
1328 	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1329 	Elf64_Sym *syms = symtab->raw_data;
1330 
1331 	return &syms[sym_idx];
1332 }
1333 
find_glob_sym(struct bpf_linker * linker,const char * sym_name)1334 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1335 {
1336 	struct glob_sym *glob_sym;
1337 	const char *name;
1338 	int i;
1339 
1340 	for (i = 0; i < linker->glob_sym_cnt; i++) {
1341 		glob_sym = &linker->glob_syms[i];
1342 		name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1343 
1344 		if (strcmp(name, sym_name) == 0)
1345 			return glob_sym;
1346 	}
1347 
1348 	return NULL;
1349 }
1350 
add_glob_sym(struct bpf_linker * linker)1351 static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1352 {
1353 	struct glob_sym *syms, *sym;
1354 
1355 	syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1356 				   sizeof(*linker->glob_syms));
1357 	if (!syms)
1358 		return NULL;
1359 
1360 	sym = &syms[linker->glob_sym_cnt];
1361 	memset(sym, 0, sizeof(*sym));
1362 	sym->var_idx = -1;
1363 
1364 	linker->glob_syms = syms;
1365 	linker->glob_sym_cnt++;
1366 
1367 	return sym;
1368 }
1369 
glob_sym_btf_matches(const char * sym_name,bool exact,const struct btf * btf1,__u32 id1,const struct btf * btf2,__u32 id2)1370 static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1371 				 const struct btf *btf1, __u32 id1,
1372 				 const struct btf *btf2, __u32 id2)
1373 {
1374 	const struct btf_type *t1, *t2;
1375 	bool is_static1, is_static2;
1376 	const char *n1, *n2;
1377 	int i, n;
1378 
1379 recur:
1380 	n1 = n2 = NULL;
1381 	t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1382 	t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1383 
1384 	/* check if only one side is FWD, otherwise handle with common logic */
1385 	if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1386 		n1 = btf__str_by_offset(btf1, t1->name_off);
1387 		n2 = btf__str_by_offset(btf2, t2->name_off);
1388 		if (strcmp(n1, n2) != 0) {
1389 			pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1390 				sym_name, n1, n2);
1391 			return false;
1392 		}
1393 		/* validate if FWD kind matches concrete kind */
1394 		if (btf_is_fwd(t1)) {
1395 			if (btf_kflag(t1) && btf_is_union(t2))
1396 				return true;
1397 			if (!btf_kflag(t1) && btf_is_struct(t2))
1398 				return true;
1399 			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1400 				sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1401 		} else {
1402 			if (btf_kflag(t2) && btf_is_union(t1))
1403 				return true;
1404 			if (!btf_kflag(t2) && btf_is_struct(t1))
1405 				return true;
1406 			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1407 				sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1408 		}
1409 		return false;
1410 	}
1411 
1412 	if (btf_kind(t1) != btf_kind(t2)) {
1413 		pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1414 			sym_name, btf_kind_str(t1), btf_kind_str(t2));
1415 		return false;
1416 	}
1417 
1418 	switch (btf_kind(t1)) {
1419 	case BTF_KIND_STRUCT:
1420 	case BTF_KIND_UNION:
1421 	case BTF_KIND_ENUM:
1422 	case BTF_KIND_ENUM64:
1423 	case BTF_KIND_FWD:
1424 	case BTF_KIND_FUNC:
1425 	case BTF_KIND_VAR:
1426 		n1 = btf__str_by_offset(btf1, t1->name_off);
1427 		n2 = btf__str_by_offset(btf2, t2->name_off);
1428 		if (strcmp(n1, n2) != 0) {
1429 			pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1430 				sym_name, btf_kind_str(t1), n1, n2);
1431 			return false;
1432 		}
1433 		break;
1434 	default:
1435 		break;
1436 	}
1437 
1438 	switch (btf_kind(t1)) {
1439 	case BTF_KIND_UNKN: /* void */
1440 	case BTF_KIND_FWD:
1441 		return true;
1442 	case BTF_KIND_INT:
1443 	case BTF_KIND_FLOAT:
1444 	case BTF_KIND_ENUM:
1445 	case BTF_KIND_ENUM64:
1446 		/* ignore encoding for int and enum values for enum */
1447 		if (t1->size != t2->size) {
1448 			pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1449 				sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1450 			return false;
1451 		}
1452 		return true;
1453 	case BTF_KIND_PTR:
1454 		/* just validate overall shape of the referenced type, so no
1455 		 * contents comparison for struct/union, and allowed fwd vs
1456 		 * struct/union
1457 		 */
1458 		exact = false;
1459 		id1 = t1->type;
1460 		id2 = t2->type;
1461 		goto recur;
1462 	case BTF_KIND_ARRAY:
1463 		/* ignore index type and array size */
1464 		id1 = btf_array(t1)->type;
1465 		id2 = btf_array(t2)->type;
1466 		goto recur;
1467 	case BTF_KIND_FUNC:
1468 		/* extern and global linkages are compatible */
1469 		is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1470 		is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1471 		if (is_static1 != is_static2) {
1472 			pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1473 			return false;
1474 		}
1475 
1476 		id1 = t1->type;
1477 		id2 = t2->type;
1478 		goto recur;
1479 	case BTF_KIND_VAR:
1480 		/* extern and global linkages are compatible */
1481 		is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1482 		is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1483 		if (is_static1 != is_static2) {
1484 			pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1485 			return false;
1486 		}
1487 
1488 		id1 = t1->type;
1489 		id2 = t2->type;
1490 		goto recur;
1491 	case BTF_KIND_STRUCT:
1492 	case BTF_KIND_UNION: {
1493 		const struct btf_member *m1, *m2;
1494 
1495 		if (!exact)
1496 			return true;
1497 
1498 		if (btf_vlen(t1) != btf_vlen(t2)) {
1499 			pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1500 				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1501 			return false;
1502 		}
1503 
1504 		n = btf_vlen(t1);
1505 		m1 = btf_members(t1);
1506 		m2 = btf_members(t2);
1507 		for (i = 0; i < n; i++, m1++, m2++) {
1508 			n1 = btf__str_by_offset(btf1, m1->name_off);
1509 			n2 = btf__str_by_offset(btf2, m2->name_off);
1510 			if (strcmp(n1, n2) != 0) {
1511 				pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1512 					sym_name, i, n1, n2);
1513 				return false;
1514 			}
1515 			if (m1->offset != m2->offset) {
1516 				pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1517 					sym_name, i, n1);
1518 				return false;
1519 			}
1520 			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1521 				return false;
1522 		}
1523 
1524 		return true;
1525 	}
1526 	case BTF_KIND_FUNC_PROTO: {
1527 		const struct btf_param *m1, *m2;
1528 
1529 		if (btf_vlen(t1) != btf_vlen(t2)) {
1530 			pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1531 				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1532 			return false;
1533 		}
1534 
1535 		n = btf_vlen(t1);
1536 		m1 = btf_params(t1);
1537 		m2 = btf_params(t2);
1538 		for (i = 0; i < n; i++, m1++, m2++) {
1539 			/* ignore func arg names */
1540 			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1541 				return false;
1542 		}
1543 
1544 		/* now check return type as well */
1545 		id1 = t1->type;
1546 		id2 = t2->type;
1547 		goto recur;
1548 	}
1549 
1550 	/* skip_mods_and_typedefs() make this impossible */
1551 	case BTF_KIND_TYPEDEF:
1552 	case BTF_KIND_VOLATILE:
1553 	case BTF_KIND_CONST:
1554 	case BTF_KIND_RESTRICT:
1555 	/* DATASECs are never compared with each other */
1556 	case BTF_KIND_DATASEC:
1557 	default:
1558 		pr_warn("global '%s': unsupported BTF kind %s\n",
1559 			sym_name, btf_kind_str(t1));
1560 		return false;
1561 	}
1562 }
1563 
map_defs_match(const char * sym_name,const struct btf * main_btf,const struct btf_map_def * main_def,const struct btf_map_def * main_inner_def,const struct btf * extra_btf,const struct btf_map_def * extra_def,const struct btf_map_def * extra_inner_def)1564 static bool map_defs_match(const char *sym_name,
1565 			   const struct btf *main_btf,
1566 			   const struct btf_map_def *main_def,
1567 			   const struct btf_map_def *main_inner_def,
1568 			   const struct btf *extra_btf,
1569 			   const struct btf_map_def *extra_def,
1570 			   const struct btf_map_def *extra_inner_def)
1571 {
1572 	const char *reason;
1573 
1574 	if (main_def->map_type != extra_def->map_type) {
1575 		reason = "type";
1576 		goto mismatch;
1577 	}
1578 
1579 	/* check key type/size match */
1580 	if (main_def->key_size != extra_def->key_size) {
1581 		reason = "key_size";
1582 		goto mismatch;
1583 	}
1584 	if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1585 		reason = "key type";
1586 		goto mismatch;
1587 	}
1588 	if ((main_def->parts & MAP_DEF_KEY_TYPE)
1589 	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1590 				      main_btf, main_def->key_type_id,
1591 				      extra_btf, extra_def->key_type_id)) {
1592 		reason = "key type";
1593 		goto mismatch;
1594 	}
1595 
1596 	/* validate value type/size match */
1597 	if (main_def->value_size != extra_def->value_size) {
1598 		reason = "value_size";
1599 		goto mismatch;
1600 	}
1601 	if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1602 		reason = "value type";
1603 		goto mismatch;
1604 	}
1605 	if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1606 	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1607 				      main_btf, main_def->value_type_id,
1608 				      extra_btf, extra_def->value_type_id)) {
1609 		reason = "key type";
1610 		goto mismatch;
1611 	}
1612 
1613 	if (main_def->max_entries != extra_def->max_entries) {
1614 		reason = "max_entries";
1615 		goto mismatch;
1616 	}
1617 	if (main_def->map_flags != extra_def->map_flags) {
1618 		reason = "map_flags";
1619 		goto mismatch;
1620 	}
1621 	if (main_def->numa_node != extra_def->numa_node) {
1622 		reason = "numa_node";
1623 		goto mismatch;
1624 	}
1625 	if (main_def->pinning != extra_def->pinning) {
1626 		reason = "pinning";
1627 		goto mismatch;
1628 	}
1629 
1630 	if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1631 		reason = "inner map";
1632 		goto mismatch;
1633 	}
1634 
1635 	if (main_def->parts & MAP_DEF_INNER_MAP) {
1636 		char inner_map_name[128];
1637 
1638 		snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1639 
1640 		return map_defs_match(inner_map_name,
1641 				      main_btf, main_inner_def, NULL,
1642 				      extra_btf, extra_inner_def, NULL);
1643 	}
1644 
1645 	return true;
1646 
1647 mismatch:
1648 	pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1649 	return false;
1650 }
1651 
glob_map_defs_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,int btf_id)1652 static bool glob_map_defs_match(const char *sym_name,
1653 				struct bpf_linker *linker, struct glob_sym *glob_sym,
1654 				struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1655 {
1656 	struct btf_map_def dst_def = {}, dst_inner_def = {};
1657 	struct btf_map_def src_def = {}, src_inner_def = {};
1658 	const struct btf_type *t;
1659 	int err;
1660 
1661 	t = btf__type_by_id(obj->btf, btf_id);
1662 	if (!btf_is_var(t)) {
1663 		pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1664 		return false;
1665 	}
1666 	t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1667 
1668 	err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1669 	if (err) {
1670 		pr_warn("global '%s': invalid map definition\n", sym_name);
1671 		return false;
1672 	}
1673 
1674 	/* re-parse existing map definition */
1675 	t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1676 	t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1677 	err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1678 	if (err) {
1679 		/* this should not happen, because we already validated it */
1680 		pr_warn("global '%s': invalid dst map definition\n", sym_name);
1681 		return false;
1682 	}
1683 
1684 	/* Currently extern map definition has to be complete and match
1685 	 * concrete map definition exactly. This restriction might be lifted
1686 	 * in the future.
1687 	 */
1688 	return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1689 			      obj->btf, &src_def, &src_inner_def);
1690 }
1691 
glob_syms_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,size_t sym_idx,int btf_id)1692 static bool glob_syms_match(const char *sym_name,
1693 			    struct bpf_linker *linker, struct glob_sym *glob_sym,
1694 			    struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1695 {
1696 	const struct btf_type *src_t;
1697 
1698 	/* if we are dealing with externs, BTF types describing both global
1699 	 * and extern VARs/FUNCs should be completely present in all files
1700 	 */
1701 	if (!glob_sym->btf_id || !btf_id) {
1702 		pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1703 		return false;
1704 	}
1705 
1706 	src_t = btf__type_by_id(obj->btf, btf_id);
1707 	if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1708 		pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1709 			btf_kind_str(src_t), sym_name);
1710 		return false;
1711 	}
1712 
1713 	/* deal with .maps definitions specially */
1714 	if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1715 		return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1716 
1717 	if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1718 				  linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1719 		return false;
1720 
1721 	return true;
1722 }
1723 
btf_is_non_static(const struct btf_type * t)1724 static bool btf_is_non_static(const struct btf_type *t)
1725 {
1726 	return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1727 	       || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1728 }
1729 
find_glob_sym_btf(struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int * out_btf_sec_id,int * out_btf_id)1730 static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1731 			     int *out_btf_sec_id, int *out_btf_id)
1732 {
1733 	int i, j, n, m, btf_id = 0;
1734 	const struct btf_type *t;
1735 	const struct btf_var_secinfo *vi;
1736 	const char *name;
1737 
1738 	if (!obj->btf) {
1739 		pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1740 		return -EINVAL;
1741 	}
1742 
1743 	n = btf__type_cnt(obj->btf);
1744 	for (i = 1; i < n; i++) {
1745 		t = btf__type_by_id(obj->btf, i);
1746 
1747 		/* some global and extern FUNCs and VARs might not be associated with any
1748 		 * DATASEC, so try to detect them in the same pass
1749 		 */
1750 		if (btf_is_non_static(t)) {
1751 			name = btf__str_by_offset(obj->btf, t->name_off);
1752 			if (strcmp(name, sym_name) != 0)
1753 				continue;
1754 
1755 			/* remember and still try to find DATASEC */
1756 			btf_id = i;
1757 			continue;
1758 		}
1759 
1760 		if (!btf_is_datasec(t))
1761 			continue;
1762 
1763 		vi = btf_var_secinfos(t);
1764 		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1765 			t = btf__type_by_id(obj->btf, vi->type);
1766 			name = btf__str_by_offset(obj->btf, t->name_off);
1767 
1768 			if (strcmp(name, sym_name) != 0)
1769 				continue;
1770 			if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1771 				continue;
1772 			if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1773 				continue;
1774 
1775 			if (btf_id && btf_id != vi->type) {
1776 				pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1777 					sym_name, btf_id, vi->type);
1778 				return -EINVAL;
1779 			}
1780 
1781 			*out_btf_sec_id = i;
1782 			*out_btf_id = vi->type;
1783 
1784 			return 0;
1785 		}
1786 	}
1787 
1788 	/* free-floating extern or global FUNC */
1789 	if (btf_id) {
1790 		*out_btf_sec_id = 0;
1791 		*out_btf_id = btf_id;
1792 		return 0;
1793 	}
1794 
1795 	pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1796 	return -ENOENT;
1797 }
1798 
find_src_sec_by_name(struct src_obj * obj,const char * sec_name)1799 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1800 {
1801 	struct src_sec *sec;
1802 	int i;
1803 
1804 	for (i = 1; i < obj->sec_cnt; i++) {
1805 		sec = &obj->secs[i];
1806 
1807 		if (strcmp(sec->sec_name, sec_name) == 0)
1808 			return sec;
1809 	}
1810 
1811 	return NULL;
1812 }
1813 
complete_extern_btf_info(struct btf * dst_btf,int dst_id,struct btf * src_btf,int src_id)1814 static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1815 				    struct btf *src_btf, int src_id)
1816 {
1817 	struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1818 	struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1819 	struct btf_param *src_p, *dst_p;
1820 	const char *s;
1821 	int i, n, off;
1822 
1823 	/* We already made sure that source and destination types (FUNC or
1824 	 * VAR) match in terms of types and argument names.
1825 	 */
1826 	if (btf_is_var(dst_t)) {
1827 		btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1828 		return 0;
1829 	}
1830 
1831 	dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1832 
1833 	/* now onto FUNC_PROTO types */
1834 	src_t = btf_type_by_id(src_btf, src_t->type);
1835 	dst_t = btf_type_by_id(dst_btf, dst_t->type);
1836 
1837 	/* Fill in all the argument names, which for extern FUNCs are missing.
1838 	 * We'll end up with two copies of FUNCs/VARs for externs, but that
1839 	 * will be taken care of by BTF dedup at the very end.
1840 	 * It might be that BTF types for extern in one file has less/more BTF
1841 	 * information (e.g., FWD instead of full STRUCT/UNION information),
1842 	 * but that should be (in most cases, subject to BTF dedup rules)
1843 	 * handled and resolved by BTF dedup algorithm as well, so we won't
1844 	 * worry about it. Our only job is to make sure that argument names
1845 	 * are populated on both sides, otherwise BTF dedup will pedantically
1846 	 * consider them different.
1847 	 */
1848 	src_p = btf_params(src_t);
1849 	dst_p = btf_params(dst_t);
1850 	for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1851 		if (!src_p->name_off)
1852 			continue;
1853 
1854 		/* src_btf has more complete info, so add name to dst_btf */
1855 		s = btf__str_by_offset(src_btf, src_p->name_off);
1856 		off = btf__add_str(dst_btf, s);
1857 		if (off < 0)
1858 			return off;
1859 		dst_p->name_off = off;
1860 	}
1861 	return 0;
1862 }
1863 
sym_update_bind(Elf64_Sym * sym,int sym_bind)1864 static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1865 {
1866 	sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1867 }
1868 
sym_update_type(Elf64_Sym * sym,int sym_type)1869 static void sym_update_type(Elf64_Sym *sym, int sym_type)
1870 {
1871 	sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1872 }
1873 
sym_update_visibility(Elf64_Sym * sym,int sym_vis)1874 static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1875 {
1876 	/* libelf doesn't provide setters for ST_VISIBILITY,
1877 	 * but it is stored in the lower 2 bits of st_other
1878 	 */
1879 	sym->st_other &= ~0x03;
1880 	sym->st_other |= sym_vis;
1881 }
1882 
linker_append_elf_sym(struct bpf_linker * linker,struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int src_sym_idx)1883 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1884 				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1885 {
1886 	struct src_sec *src_sec = NULL;
1887 	struct dst_sec *dst_sec = NULL;
1888 	struct glob_sym *glob_sym = NULL;
1889 	int name_off, sym_type, sym_bind, sym_vis, err;
1890 	int btf_sec_id = 0, btf_id = 0;
1891 	size_t dst_sym_idx;
1892 	Elf64_Sym *dst_sym;
1893 	bool sym_is_extern;
1894 
1895 	sym_type = ELF64_ST_TYPE(sym->st_info);
1896 	sym_bind = ELF64_ST_BIND(sym->st_info);
1897 	sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1898 	sym_is_extern = sym->st_shndx == SHN_UNDEF;
1899 
1900 	if (sym_is_extern) {
1901 		if (!obj->btf) {
1902 			pr_warn("externs without BTF info are not supported\n");
1903 			return -ENOTSUP;
1904 		}
1905 	} else if (sym->st_shndx < SHN_LORESERVE) {
1906 		src_sec = &obj->secs[sym->st_shndx];
1907 		if (src_sec->skipped)
1908 			return 0;
1909 		dst_sec = &linker->secs[src_sec->dst_id];
1910 
1911 		/* allow only one STT_SECTION symbol per section */
1912 		if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1913 			obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1914 			return 0;
1915 		}
1916 	}
1917 
1918 	if (sym_bind == STB_LOCAL)
1919 		goto add_sym;
1920 
1921 	/* find matching BTF info */
1922 	err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1923 	if (err)
1924 		return err;
1925 
1926 	if (sym_is_extern && btf_sec_id) {
1927 		const char *sec_name = NULL;
1928 		const struct btf_type *t;
1929 
1930 		t = btf__type_by_id(obj->btf, btf_sec_id);
1931 		sec_name = btf__str_by_offset(obj->btf, t->name_off);
1932 
1933 		/* Clang puts unannotated extern vars into
1934 		 * '.extern' BTF DATASEC. Treat them the same
1935 		 * as unannotated extern funcs (which are
1936 		 * currently not put into any DATASECs).
1937 		 * Those don't have associated src_sec/dst_sec.
1938 		 */
1939 		if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1940 			src_sec = find_src_sec_by_name(obj, sec_name);
1941 			if (!src_sec) {
1942 				pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1943 				return -ENOENT;
1944 			}
1945 			dst_sec = &linker->secs[src_sec->dst_id];
1946 		}
1947 	}
1948 
1949 	glob_sym = find_glob_sym(linker, sym_name);
1950 	if (glob_sym) {
1951 		/* Preventively resolve to existing symbol. This is
1952 		 * needed for further relocation symbol remapping in
1953 		 * the next step of linking.
1954 		 */
1955 		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1956 
1957 		/* If both symbols are non-externs, at least one of
1958 		 * them has to be STB_WEAK, otherwise they are in
1959 		 * a conflict with each other.
1960 		 */
1961 		if (!sym_is_extern && !glob_sym->is_extern
1962 		    && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1963 			pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1964 				src_sym_idx, sym_name, obj->filename);
1965 			return -EINVAL;
1966 		}
1967 
1968 		if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1969 			return -EINVAL;
1970 
1971 		dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1972 
1973 		/* If new symbol is strong, then force dst_sym to be strong as
1974 		 * well; this way a mix of weak and non-weak extern
1975 		 * definitions will end up being strong.
1976 		 */
1977 		if (sym_bind == STB_GLOBAL) {
1978 			/* We still need to preserve type (NOTYPE or
1979 			 * OBJECT/FUNC, depending on whether the symbol is
1980 			 * extern or not)
1981 			 */
1982 			sym_update_bind(dst_sym, STB_GLOBAL);
1983 			glob_sym->is_weak = false;
1984 		}
1985 
1986 		/* Non-default visibility is "contaminating", with stricter
1987 		 * visibility overwriting more permissive ones, even if more
1988 		 * permissive visibility comes from just an extern definition.
1989 		 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1990 		 * ensured by ELF symbol sanity checks above.
1991 		 */
1992 		if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1993 			sym_update_visibility(dst_sym, sym_vis);
1994 
1995 		/* If the new symbol is extern, then regardless if
1996 		 * existing symbol is extern or resolved global, just
1997 		 * keep the existing one untouched.
1998 		 */
1999 		if (sym_is_extern)
2000 			return 0;
2001 
2002 		/* If existing symbol is a strong resolved symbol, bail out,
2003 		 * because we lost resolution battle have nothing to
2004 		 * contribute. We already checked above that there is no
2005 		 * strong-strong conflict. We also already tightened binding
2006 		 * and visibility, so nothing else to contribute at that point.
2007 		 */
2008 		if (!glob_sym->is_extern && sym_bind == STB_WEAK)
2009 			return 0;
2010 
2011 		/* At this point, new symbol is strong non-extern,
2012 		 * so overwrite glob_sym with new symbol information.
2013 		 * Preserve binding and visibility.
2014 		 */
2015 		sym_update_type(dst_sym, sym_type);
2016 		dst_sym->st_shndx = dst_sec->sec_idx;
2017 		dst_sym->st_value = src_sec->dst_off + sym->st_value;
2018 		dst_sym->st_size = sym->st_size;
2019 
2020 		/* see comment below about dst_sec->id vs dst_sec->sec_idx */
2021 		glob_sym->sec_id = dst_sec->id;
2022 		glob_sym->is_extern = false;
2023 
2024 		if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
2025 					     obj->btf, btf_id))
2026 			return -EINVAL;
2027 
2028 		/* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
2029 		glob_sym->underlying_btf_id = 0;
2030 
2031 		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2032 		return 0;
2033 	}
2034 
2035 add_sym:
2036 	name_off = strset__add_str(linker->strtab_strs, sym_name);
2037 	if (name_off < 0)
2038 		return name_off;
2039 
2040 	dst_sym = add_new_sym(linker, &dst_sym_idx);
2041 	if (!dst_sym)
2042 		return -ENOMEM;
2043 
2044 	dst_sym->st_name = name_off;
2045 	dst_sym->st_info = sym->st_info;
2046 	dst_sym->st_other = sym->st_other;
2047 	dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
2048 	dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
2049 	dst_sym->st_size = sym->st_size;
2050 
2051 	obj->sym_map[src_sym_idx] = dst_sym_idx;
2052 
2053 	if (sym_type == STT_SECTION && dst_sym) {
2054 		dst_sec->sec_sym_idx = dst_sym_idx;
2055 		dst_sym->st_value = 0;
2056 	}
2057 
2058 	if (sym_bind != STB_LOCAL) {
2059 		glob_sym = add_glob_sym(linker);
2060 		if (!glob_sym)
2061 			return -ENOMEM;
2062 
2063 		glob_sym->sym_idx = dst_sym_idx;
2064 		/* we use dst_sec->id (and not dst_sec->sec_idx), because
2065 		 * ephemeral sections (.kconfig, .ksyms, etc) don't have
2066 		 * sec_idx (as they don't have corresponding ELF section), but
2067 		 * still have id. .extern doesn't have even ephemeral section
2068 		 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
2069 		 */
2070 		glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
2071 		glob_sym->name_off = name_off;
2072 		/* we will fill btf_id in during BTF merging step */
2073 		glob_sym->btf_id = 0;
2074 		glob_sym->is_extern = sym_is_extern;
2075 		glob_sym->is_weak = sym_bind == STB_WEAK;
2076 	}
2077 
2078 	return 0;
2079 }
2080 
linker_append_elf_relos(struct bpf_linker * linker,struct src_obj * obj)2081 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2082 {
2083 	struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2084 	int i, err;
2085 
2086 	for (i = 1; i < obj->sec_cnt; i++) {
2087 		struct src_sec *src_sec, *src_linked_sec;
2088 		struct dst_sec *dst_sec, *dst_linked_sec;
2089 		Elf64_Rel *src_rel, *dst_rel;
2090 		int j, n;
2091 
2092 		src_sec = &obj->secs[i];
2093 		if (!is_relo_sec(src_sec))
2094 			continue;
2095 
2096 		/* shdr->sh_info points to relocatable section */
2097 		src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2098 		if (src_linked_sec->skipped)
2099 			continue;
2100 
2101 		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2102 		if (!dst_sec) {
2103 			dst_sec = add_dst_sec(linker, src_sec->sec_name);
2104 			if (!dst_sec)
2105 				return -ENOMEM;
2106 			err = init_sec(linker, dst_sec, src_sec);
2107 			if (err) {
2108 				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2109 				return err;
2110 			}
2111 		} else if (!secs_match(dst_sec, src_sec)) {
2112 			pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2113 			return -1;
2114 		}
2115 
2116 		/* shdr->sh_link points to SYMTAB */
2117 		dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2118 
2119 		/* shdr->sh_info points to relocated section */
2120 		dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2121 		dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2122 
2123 		src_sec->dst_id = dst_sec->id;
2124 		err = extend_sec(linker, dst_sec, src_sec);
2125 		if (err)
2126 			return err;
2127 
2128 		src_rel = src_sec->data->d_buf;
2129 		dst_rel = dst_sec->raw_data + src_sec->dst_off;
2130 		n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2131 		for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2132 			size_t src_sym_idx, dst_sym_idx, sym_type;
2133 			Elf64_Sym *src_sym;
2134 
2135 			src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2136 			src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2137 
2138 			dst_sym_idx = obj->sym_map[src_sym_idx];
2139 			dst_rel->r_offset += src_linked_sec->dst_off;
2140 			sym_type = ELF64_R_TYPE(src_rel->r_info);
2141 			dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2142 
2143 			if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2144 				struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2145 				struct bpf_insn *insn;
2146 
2147 				if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2148 					/* calls to the very first static function inside
2149 					 * .text section at offset 0 will
2150 					 * reference section symbol, not the
2151 					 * function symbol. Fix that up,
2152 					 * otherwise it won't be possible to
2153 					 * relocate calls to two different
2154 					 * static functions with the same name
2155 					 * (rom two different object files)
2156 					 */
2157 					insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2158 					if (insn->code == (BPF_JMP | BPF_CALL))
2159 						insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2160 					else
2161 						insn->imm += sec->dst_off;
2162 				} else {
2163 					pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2164 					return -EINVAL;
2165 				}
2166 			}
2167 
2168 		}
2169 	}
2170 
2171 	return 0;
2172 }
2173 
find_sym_by_name(struct src_obj * obj,size_t sec_idx,int sym_type,const char * sym_name)2174 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2175 				   int sym_type, const char *sym_name)
2176 {
2177 	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2178 	Elf64_Sym *sym = symtab->data->d_buf;
2179 	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2180 	int str_sec_idx = symtab->shdr->sh_link;
2181 	const char *name;
2182 
2183 	for (i = 0; i < n; i++, sym++) {
2184 		if (sym->st_shndx != sec_idx)
2185 			continue;
2186 		if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2187 			continue;
2188 
2189 		name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2190 		if (!name)
2191 			return NULL;
2192 
2193 		if (strcmp(sym_name, name) != 0)
2194 			continue;
2195 
2196 		return sym;
2197 	}
2198 
2199 	return NULL;
2200 }
2201 
linker_fixup_btf(struct src_obj * obj)2202 static int linker_fixup_btf(struct src_obj *obj)
2203 {
2204 	const char *sec_name;
2205 	struct src_sec *sec;
2206 	int i, j, n, m;
2207 
2208 	if (!obj->btf)
2209 		return 0;
2210 
2211 	n = btf__type_cnt(obj->btf);
2212 	for (i = 1; i < n; i++) {
2213 		struct btf_var_secinfo *vi;
2214 		struct btf_type *t;
2215 
2216 		t = btf_type_by_id(obj->btf, i);
2217 		if (btf_kind(t) != BTF_KIND_DATASEC)
2218 			continue;
2219 
2220 		sec_name = btf__str_by_offset(obj->btf, t->name_off);
2221 		sec = find_src_sec_by_name(obj, sec_name);
2222 		if (sec) {
2223 			/* record actual section size, unless ephemeral */
2224 			if (sec->shdr)
2225 				t->size = sec->shdr->sh_size;
2226 		} else {
2227 			/* BTF can have some sections that are not represented
2228 			 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2229 			 * for special extern variables.
2230 			 *
2231 			 * For all but one such special (ephemeral)
2232 			 * sections, we pre-create "section shells" to be able
2233 			 * to keep track of extra per-section metadata later
2234 			 * (e.g., those BTF extern variables).
2235 			 *
2236 			 * .extern is even more special, though, because it
2237 			 * contains extern variables that need to be resolved
2238 			 * by static linker, not libbpf and kernel. When such
2239 			 * externs are resolved, we are going to remove them
2240 			 * from .extern BTF section and might end up not
2241 			 * needing it at all. Each resolved extern should have
2242 			 * matching non-extern VAR/FUNC in other sections.
2243 			 *
2244 			 * We do support leaving some of the externs
2245 			 * unresolved, though, to support cases of building
2246 			 * libraries, which will later be linked against final
2247 			 * BPF applications. So if at finalization we still
2248 			 * see unresolved externs, we'll create .extern
2249 			 * section on our own.
2250 			 */
2251 			if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2252 				continue;
2253 
2254 			sec = add_src_sec(obj, sec_name);
2255 			if (!sec)
2256 				return -ENOMEM;
2257 
2258 			sec->ephemeral = true;
2259 			sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2260 		}
2261 
2262 		/* remember ELF section and its BTF type ID match */
2263 		sec->sec_type_id = i;
2264 
2265 		/* fix up variable offsets */
2266 		vi = btf_var_secinfos(t);
2267 		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2268 			const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2269 			const char *var_name;
2270 			int var_linkage;
2271 			Elf64_Sym *sym;
2272 
2273 			/* could be a variable or function */
2274 			if (!btf_is_var(vt))
2275 				continue;
2276 
2277 			var_name = btf__str_by_offset(obj->btf, vt->name_off);
2278 			var_linkage = btf_var(vt)->linkage;
2279 
2280 			/* no need to patch up static or extern vars */
2281 			if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2282 				continue;
2283 
2284 			sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2285 			if (!sym) {
2286 				pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2287 				return -ENOENT;
2288 			}
2289 
2290 			vi->offset = sym->st_value;
2291 		}
2292 	}
2293 
2294 	return 0;
2295 }
2296 
linker_append_btf(struct bpf_linker * linker,struct src_obj * obj)2297 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2298 {
2299 	const struct btf_type *t;
2300 	int i, j, n, start_id, id, err;
2301 	const char *name;
2302 
2303 	if (!obj->btf)
2304 		return 0;
2305 
2306 	start_id = btf__type_cnt(linker->btf);
2307 	n = btf__type_cnt(obj->btf);
2308 
2309 	obj->btf_type_map = calloc(n + 1, sizeof(int));
2310 	if (!obj->btf_type_map)
2311 		return -ENOMEM;
2312 
2313 	for (i = 1; i < n; i++) {
2314 		struct glob_sym *glob_sym = NULL;
2315 
2316 		t = btf__type_by_id(obj->btf, i);
2317 
2318 		/* DATASECs are handled specially below */
2319 		if (btf_kind(t) == BTF_KIND_DATASEC)
2320 			continue;
2321 
2322 		if (btf_is_non_static(t)) {
2323 			/* there should be glob_sym already */
2324 			name = btf__str_by_offset(obj->btf, t->name_off);
2325 			glob_sym = find_glob_sym(linker, name);
2326 
2327 			/* VARs without corresponding glob_sym are those that
2328 			 * belong to skipped/deduplicated sections (i.e.,
2329 			 * license and version), so just skip them
2330 			 */
2331 			if (!glob_sym)
2332 				continue;
2333 
2334 			/* linker_append_elf_sym() might have requested
2335 			 * updating underlying type ID, if extern was resolved
2336 			 * to strong symbol or weak got upgraded to non-weak
2337 			 */
2338 			if (glob_sym->underlying_btf_id == 0)
2339 				glob_sym->underlying_btf_id = -t->type;
2340 
2341 			/* globals from previous object files that match our
2342 			 * VAR/FUNC already have a corresponding associated
2343 			 * BTF type, so just make sure to use it
2344 			 */
2345 			if (glob_sym->btf_id) {
2346 				/* reuse existing BTF type for global var/func */
2347 				obj->btf_type_map[i] = glob_sym->btf_id;
2348 				continue;
2349 			}
2350 		}
2351 
2352 		id = btf__add_type(linker->btf, obj->btf, t);
2353 		if (id < 0) {
2354 			pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2355 			return id;
2356 		}
2357 
2358 		obj->btf_type_map[i] = id;
2359 
2360 		/* record just appended BTF type for var/func */
2361 		if (glob_sym) {
2362 			glob_sym->btf_id = id;
2363 			glob_sym->underlying_btf_id = -t->type;
2364 		}
2365 	}
2366 
2367 	/* remap all the types except DATASECs */
2368 	n = btf__type_cnt(linker->btf);
2369 	for (i = start_id; i < n; i++) {
2370 		struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2371 		struct btf_field_iter it;
2372 		__u32 *type_id;
2373 
2374 		err = btf_field_iter_init(&it, dst_t, BTF_FIELD_ITER_IDS);
2375 		if (err)
2376 			return err;
2377 
2378 		while ((type_id = btf_field_iter_next(&it))) {
2379 			int new_id = obj->btf_type_map[*type_id];
2380 
2381 			/* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2382 			if (new_id == 0 && *type_id != 0) {
2383 				pr_warn("failed to find new ID mapping for original BTF type ID %u\n",
2384 					*type_id);
2385 				return -EINVAL;
2386 			}
2387 
2388 			*type_id = obj->btf_type_map[*type_id];
2389 		}
2390 	}
2391 
2392 	/* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2393 	 * actual type), if necessary
2394 	 */
2395 	for (i = 0; i < linker->glob_sym_cnt; i++) {
2396 		struct glob_sym *glob_sym = &linker->glob_syms[i];
2397 		struct btf_type *glob_t;
2398 
2399 		if (glob_sym->underlying_btf_id >= 0)
2400 			continue;
2401 
2402 		glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2403 
2404 		glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2405 		glob_t->type = glob_sym->underlying_btf_id;
2406 	}
2407 
2408 	/* append DATASEC info */
2409 	for (i = 1; i < obj->sec_cnt; i++) {
2410 		struct src_sec *src_sec;
2411 		struct dst_sec *dst_sec;
2412 		const struct btf_var_secinfo *src_var;
2413 		struct btf_var_secinfo *dst_var;
2414 
2415 		src_sec = &obj->secs[i];
2416 		if (!src_sec->sec_type_id || src_sec->skipped)
2417 			continue;
2418 		dst_sec = &linker->secs[src_sec->dst_id];
2419 
2420 		/* Mark section as having BTF regardless of the presence of
2421 		 * variables. In some cases compiler might generate empty BTF
2422 		 * with no variables information. E.g., when promoting local
2423 		 * array/structure variable initial values and BPF object
2424 		 * file otherwise has no read-only static variables in
2425 		 * .rodata. We need to preserve such empty BTF and just set
2426 		 * correct section size.
2427 		 */
2428 		dst_sec->has_btf = true;
2429 
2430 		t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2431 		src_var = btf_var_secinfos(t);
2432 		n = btf_vlen(t);
2433 		for (j = 0; j < n; j++, src_var++) {
2434 			void *sec_vars = dst_sec->sec_vars;
2435 			int new_id = obj->btf_type_map[src_var->type];
2436 			struct glob_sym *glob_sym = NULL;
2437 
2438 			t = btf_type_by_id(linker->btf, new_id);
2439 			if (btf_is_non_static(t)) {
2440 				name = btf__str_by_offset(linker->btf, t->name_off);
2441 				glob_sym = find_glob_sym(linker, name);
2442 				if (glob_sym->sec_id != dst_sec->id) {
2443 					pr_warn("global '%s': section mismatch %d vs %d\n",
2444 						name, glob_sym->sec_id, dst_sec->id);
2445 					return -EINVAL;
2446 				}
2447 			}
2448 
2449 			/* If there is already a member (VAR or FUNC) mapped
2450 			 * to the same type, don't add a duplicate entry.
2451 			 * This will happen when multiple object files define
2452 			 * the same extern VARs/FUNCs.
2453 			 */
2454 			if (glob_sym && glob_sym->var_idx >= 0) {
2455 				__s64 sz;
2456 
2457 				/* FUNCs don't have size, nothing to update */
2458 				if (btf_is_func(t))
2459 					continue;
2460 
2461 				dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2462 				/* Because underlying BTF type might have
2463 				 * changed, so might its size have changed, so
2464 				 * re-calculate and update it in sec_var.
2465 				 */
2466 				sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2467 				if (sz < 0) {
2468 					pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2469 						name, (int)sz);
2470 					return -EINVAL;
2471 				}
2472 				dst_var->size = sz;
2473 				continue;
2474 			}
2475 
2476 			sec_vars = libbpf_reallocarray(sec_vars,
2477 						       dst_sec->sec_var_cnt + 1,
2478 						       sizeof(*dst_sec->sec_vars));
2479 			if (!sec_vars)
2480 				return -ENOMEM;
2481 
2482 			dst_sec->sec_vars = sec_vars;
2483 			dst_sec->sec_var_cnt++;
2484 
2485 			dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2486 			dst_var->type = obj->btf_type_map[src_var->type];
2487 			dst_var->size = src_var->size;
2488 			dst_var->offset = src_sec->dst_off + src_var->offset;
2489 
2490 			if (glob_sym)
2491 				glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2492 		}
2493 	}
2494 
2495 	return 0;
2496 }
2497 
add_btf_ext_rec(struct btf_ext_sec_data * ext_data,const void * src_rec)2498 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2499 {
2500 	void *tmp;
2501 
2502 	tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2503 	if (!tmp)
2504 		return NULL;
2505 	ext_data->recs = tmp;
2506 
2507 	tmp += ext_data->rec_cnt * ext_data->rec_sz;
2508 	memcpy(tmp, src_rec, ext_data->rec_sz);
2509 
2510 	ext_data->rec_cnt++;
2511 
2512 	return tmp;
2513 }
2514 
linker_append_btf_ext(struct bpf_linker * linker,struct src_obj * obj)2515 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2516 {
2517 	const struct btf_ext_info_sec *ext_sec;
2518 	const char *sec_name, *s;
2519 	struct src_sec *src_sec;
2520 	struct dst_sec *dst_sec;
2521 	int rec_sz, str_off, i;
2522 
2523 	if (!obj->btf_ext)
2524 		return 0;
2525 
2526 	rec_sz = obj->btf_ext->func_info.rec_size;
2527 	for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2528 		struct bpf_func_info_min *src_rec, *dst_rec;
2529 
2530 		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2531 		src_sec = find_src_sec_by_name(obj, sec_name);
2532 		if (!src_sec) {
2533 			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2534 			return -EINVAL;
2535 		}
2536 		dst_sec = &linker->secs[src_sec->dst_id];
2537 
2538 		if (dst_sec->func_info.rec_sz == 0)
2539 			dst_sec->func_info.rec_sz = rec_sz;
2540 		if (dst_sec->func_info.rec_sz != rec_sz) {
2541 			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2542 			return -EINVAL;
2543 		}
2544 
2545 		for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2546 			dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2547 			if (!dst_rec)
2548 				return -ENOMEM;
2549 
2550 			dst_rec->insn_off += src_sec->dst_off;
2551 			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2552 		}
2553 	}
2554 
2555 	rec_sz = obj->btf_ext->line_info.rec_size;
2556 	for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2557 		struct bpf_line_info_min *src_rec, *dst_rec;
2558 
2559 		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2560 		src_sec = find_src_sec_by_name(obj, sec_name);
2561 		if (!src_sec) {
2562 			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2563 			return -EINVAL;
2564 		}
2565 		dst_sec = &linker->secs[src_sec->dst_id];
2566 
2567 		if (dst_sec->line_info.rec_sz == 0)
2568 			dst_sec->line_info.rec_sz = rec_sz;
2569 		if (dst_sec->line_info.rec_sz != rec_sz) {
2570 			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2571 			return -EINVAL;
2572 		}
2573 
2574 		for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2575 			dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2576 			if (!dst_rec)
2577 				return -ENOMEM;
2578 
2579 			dst_rec->insn_off += src_sec->dst_off;
2580 
2581 			s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2582 			str_off = btf__add_str(linker->btf, s);
2583 			if (str_off < 0)
2584 				return -ENOMEM;
2585 			dst_rec->file_name_off = str_off;
2586 
2587 			s = btf__str_by_offset(obj->btf, src_rec->line_off);
2588 			str_off = btf__add_str(linker->btf, s);
2589 			if (str_off < 0)
2590 				return -ENOMEM;
2591 			dst_rec->line_off = str_off;
2592 
2593 			/* dst_rec->line_col is fine */
2594 		}
2595 	}
2596 
2597 	rec_sz = obj->btf_ext->core_relo_info.rec_size;
2598 	for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2599 		struct bpf_core_relo *src_rec, *dst_rec;
2600 
2601 		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2602 		src_sec = find_src_sec_by_name(obj, sec_name);
2603 		if (!src_sec) {
2604 			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2605 			return -EINVAL;
2606 		}
2607 		dst_sec = &linker->secs[src_sec->dst_id];
2608 
2609 		if (dst_sec->core_relo_info.rec_sz == 0)
2610 			dst_sec->core_relo_info.rec_sz = rec_sz;
2611 		if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2612 			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2613 			return -EINVAL;
2614 		}
2615 
2616 		for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2617 			dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2618 			if (!dst_rec)
2619 				return -ENOMEM;
2620 
2621 			dst_rec->insn_off += src_sec->dst_off;
2622 			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2623 
2624 			s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2625 			str_off = btf__add_str(linker->btf, s);
2626 			if (str_off < 0)
2627 				return -ENOMEM;
2628 			dst_rec->access_str_off = str_off;
2629 
2630 			/* dst_rec->kind is fine */
2631 		}
2632 	}
2633 
2634 	return 0;
2635 }
2636 
bpf_linker__finalize(struct bpf_linker * linker)2637 int bpf_linker__finalize(struct bpf_linker *linker)
2638 {
2639 	struct dst_sec *sec;
2640 	size_t strs_sz;
2641 	const void *strs;
2642 	int err, i;
2643 
2644 	if (!linker->elf)
2645 		return libbpf_err(-EINVAL);
2646 
2647 	err = finalize_btf(linker);
2648 	if (err)
2649 		return libbpf_err(err);
2650 
2651 	/* Finalize strings */
2652 	strs_sz = strset__data_size(linker->strtab_strs);
2653 	strs = strset__data(linker->strtab_strs);
2654 
2655 	sec = &linker->secs[linker->strtab_sec_idx];
2656 	sec->data->d_align = 1;
2657 	sec->data->d_off = 0LL;
2658 	sec->data->d_buf = (void *)strs;
2659 	sec->data->d_type = ELF_T_BYTE;
2660 	sec->data->d_size = strs_sz;
2661 	sec->shdr->sh_size = strs_sz;
2662 
2663 	for (i = 1; i < linker->sec_cnt; i++) {
2664 		sec = &linker->secs[i];
2665 
2666 		/* STRTAB is handled specially above */
2667 		if (sec->sec_idx == linker->strtab_sec_idx)
2668 			continue;
2669 
2670 		/* special ephemeral sections (.ksyms, .kconfig, etc) */
2671 		if (!sec->scn)
2672 			continue;
2673 
2674 		/* restore sections with bpf insns to target byte-order */
2675 		if (linker->swapped_endian && is_exec_sec(sec))
2676 			exec_sec_bswap(sec->raw_data, sec->sec_sz);
2677 
2678 		sec->data->d_buf = sec->raw_data;
2679 	}
2680 
2681 	/* Finalize ELF layout */
2682 	if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2683 		err = -errno;
2684 		pr_warn_elf("failed to finalize ELF layout");
2685 		return libbpf_err(err);
2686 	}
2687 
2688 	/* Write out final ELF contents */
2689 	if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2690 		err = -errno;
2691 		pr_warn_elf("failed to write ELF contents");
2692 		return libbpf_err(err);
2693 	}
2694 
2695 	elf_end(linker->elf);
2696 	close(linker->fd);
2697 
2698 	linker->elf = NULL;
2699 	linker->fd = -1;
2700 
2701 	return 0;
2702 }
2703 
emit_elf_data_sec(struct bpf_linker * linker,const char * sec_name,size_t align,const void * raw_data,size_t raw_sz)2704 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2705 			     size_t align, const void *raw_data, size_t raw_sz)
2706 {
2707 	Elf_Scn *scn;
2708 	Elf_Data *data;
2709 	Elf64_Shdr *shdr;
2710 	int name_off;
2711 
2712 	name_off = strset__add_str(linker->strtab_strs, sec_name);
2713 	if (name_off < 0)
2714 		return name_off;
2715 
2716 	scn = elf_newscn(linker->elf);
2717 	if (!scn)
2718 		return -ENOMEM;
2719 	data = elf_newdata(scn);
2720 	if (!data)
2721 		return -ENOMEM;
2722 	shdr = elf64_getshdr(scn);
2723 	if (!shdr)
2724 		return -EINVAL;
2725 
2726 	shdr->sh_name = name_off;
2727 	shdr->sh_type = SHT_PROGBITS;
2728 	shdr->sh_flags = 0;
2729 	shdr->sh_size = raw_sz;
2730 	shdr->sh_link = 0;
2731 	shdr->sh_info = 0;
2732 	shdr->sh_addralign = align;
2733 	shdr->sh_entsize = 0;
2734 
2735 	data->d_type = ELF_T_BYTE;
2736 	data->d_size = raw_sz;
2737 	data->d_buf = (void *)raw_data;
2738 	data->d_align = align;
2739 	data->d_off = 0;
2740 
2741 	return 0;
2742 }
2743 
finalize_btf(struct bpf_linker * linker)2744 static int finalize_btf(struct bpf_linker *linker)
2745 {
2746 	enum btf_endianness link_endianness;
2747 	LIBBPF_OPTS(btf_dedup_opts, opts);
2748 	struct btf *btf = linker->btf;
2749 	const void *raw_data;
2750 	int i, j, id, err;
2751 	__u32 raw_sz;
2752 
2753 	/* bail out if no BTF data was produced */
2754 	if (btf__type_cnt(linker->btf) == 1)
2755 		return 0;
2756 
2757 	for (i = 1; i < linker->sec_cnt; i++) {
2758 		struct dst_sec *sec = &linker->secs[i];
2759 
2760 		if (!sec->has_btf)
2761 			continue;
2762 
2763 		id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2764 		if (id < 0) {
2765 			pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2766 				sec->sec_name, id);
2767 			return id;
2768 		}
2769 
2770 		for (j = 0; j < sec->sec_var_cnt; j++) {
2771 			struct btf_var_secinfo *vi = &sec->sec_vars[j];
2772 
2773 			if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2774 				return -EINVAL;
2775 		}
2776 	}
2777 
2778 	err = finalize_btf_ext(linker);
2779 	if (err) {
2780 		pr_warn(".BTF.ext generation failed: %s\n", errstr(err));
2781 		return err;
2782 	}
2783 
2784 	opts.btf_ext = linker->btf_ext;
2785 	err = btf__dedup(linker->btf, &opts);
2786 	if (err) {
2787 		pr_warn("BTF dedup failed: %s\n", errstr(err));
2788 		return err;
2789 	}
2790 
2791 	/* Set .BTF and .BTF.ext output byte order */
2792 	link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ?
2793 			  BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
2794 	btf__set_endianness(linker->btf, link_endianness);
2795 	if (linker->btf_ext)
2796 		btf_ext__set_endianness(linker->btf_ext, link_endianness);
2797 
2798 	/* Emit .BTF section */
2799 	raw_data = btf__raw_data(linker->btf, &raw_sz);
2800 	if (!raw_data)
2801 		return -ENOMEM;
2802 
2803 	err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2804 	if (err) {
2805 		pr_warn("failed to write out .BTF ELF section: %s\n", errstr(err));
2806 		return err;
2807 	}
2808 
2809 	/* Emit .BTF.ext section */
2810 	if (linker->btf_ext) {
2811 		raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);
2812 		if (!raw_data)
2813 			return -ENOMEM;
2814 
2815 		err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2816 		if (err) {
2817 			pr_warn("failed to write out .BTF.ext ELF section: %s\n", errstr(err));
2818 			return err;
2819 		}
2820 	}
2821 
2822 	return 0;
2823 }
2824 
emit_btf_ext_data(struct bpf_linker * linker,void * output,const char * sec_name,struct btf_ext_sec_data * sec_data)2825 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2826 			     const char *sec_name, struct btf_ext_sec_data *sec_data)
2827 {
2828 	struct btf_ext_info_sec *sec_info;
2829 	void *cur = output;
2830 	int str_off;
2831 	size_t sz;
2832 
2833 	if (!sec_data->rec_cnt)
2834 		return 0;
2835 
2836 	str_off = btf__add_str(linker->btf, sec_name);
2837 	if (str_off < 0)
2838 		return -ENOMEM;
2839 
2840 	sec_info = cur;
2841 	sec_info->sec_name_off = str_off;
2842 	sec_info->num_info = sec_data->rec_cnt;
2843 	cur += sizeof(struct btf_ext_info_sec);
2844 
2845 	sz = sec_data->rec_cnt * sec_data->rec_sz;
2846 	memcpy(cur, sec_data->recs, sz);
2847 	cur += sz;
2848 
2849 	return cur - output;
2850 }
2851 
finalize_btf_ext(struct bpf_linker * linker)2852 static int finalize_btf_ext(struct bpf_linker *linker)
2853 {
2854 	size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2855 	size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2856 	struct btf_ext_header *hdr;
2857 	void *data, *cur;
2858 	int i, err, sz;
2859 
2860 	/* validate that all sections have the same .BTF.ext record sizes
2861 	 * and calculate total data size for each type of data (func info,
2862 	 * line info, core relos)
2863 	 */
2864 	for (i = 1; i < linker->sec_cnt; i++) {
2865 		struct dst_sec *sec = &linker->secs[i];
2866 
2867 		if (sec->func_info.rec_cnt) {
2868 			if (func_rec_sz == 0)
2869 				func_rec_sz = sec->func_info.rec_sz;
2870 			if (func_rec_sz != sec->func_info.rec_sz) {
2871 				pr_warn("mismatch in func_info record size %zu != %u\n",
2872 					func_rec_sz, sec->func_info.rec_sz);
2873 				return -EINVAL;
2874 			}
2875 
2876 			funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2877 		}
2878 		if (sec->line_info.rec_cnt) {
2879 			if (line_rec_sz == 0)
2880 				line_rec_sz = sec->line_info.rec_sz;
2881 			if (line_rec_sz != sec->line_info.rec_sz) {
2882 				pr_warn("mismatch in line_info record size %zu != %u\n",
2883 					line_rec_sz, sec->line_info.rec_sz);
2884 				return -EINVAL;
2885 			}
2886 
2887 			lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2888 		}
2889 		if (sec->core_relo_info.rec_cnt) {
2890 			if (core_relo_rec_sz == 0)
2891 				core_relo_rec_sz = sec->core_relo_info.rec_sz;
2892 			if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2893 				pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2894 					core_relo_rec_sz, sec->core_relo_info.rec_sz);
2895 				return -EINVAL;
2896 			}
2897 
2898 			core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2899 		}
2900 	}
2901 
2902 	if (!funcs_sz && !lines_sz && !core_relos_sz)
2903 		return 0;
2904 
2905 	total_sz += sizeof(struct btf_ext_header);
2906 	if (funcs_sz) {
2907 		funcs_sz += sizeof(__u32); /* record size prefix */
2908 		total_sz += funcs_sz;
2909 	}
2910 	if (lines_sz) {
2911 		lines_sz += sizeof(__u32); /* record size prefix */
2912 		total_sz += lines_sz;
2913 	}
2914 	if (core_relos_sz) {
2915 		core_relos_sz += sizeof(__u32); /* record size prefix */
2916 		total_sz += core_relos_sz;
2917 	}
2918 
2919 	cur = data = calloc(1, total_sz);
2920 	if (!data)
2921 		return -ENOMEM;
2922 
2923 	hdr = cur;
2924 	hdr->magic = BTF_MAGIC;
2925 	hdr->version = BTF_VERSION;
2926 	hdr->flags = 0;
2927 	hdr->hdr_len = sizeof(struct btf_ext_header);
2928 	cur += sizeof(struct btf_ext_header);
2929 
2930 	/* All offsets are in bytes relative to the end of this header */
2931 	hdr->func_info_off = 0;
2932 	hdr->func_info_len = funcs_sz;
2933 	hdr->line_info_off = funcs_sz;
2934 	hdr->line_info_len = lines_sz;
2935 	hdr->core_relo_off = funcs_sz + lines_sz;
2936 	hdr->core_relo_len = core_relos_sz;
2937 
2938 	if (funcs_sz) {
2939 		*(__u32 *)cur = func_rec_sz;
2940 		cur += sizeof(__u32);
2941 
2942 		for (i = 1; i < linker->sec_cnt; i++) {
2943 			struct dst_sec *sec = &linker->secs[i];
2944 
2945 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2946 			if (sz < 0) {
2947 				err = sz;
2948 				goto out;
2949 			}
2950 
2951 			cur += sz;
2952 		}
2953 	}
2954 
2955 	if (lines_sz) {
2956 		*(__u32 *)cur = line_rec_sz;
2957 		cur += sizeof(__u32);
2958 
2959 		for (i = 1; i < linker->sec_cnt; i++) {
2960 			struct dst_sec *sec = &linker->secs[i];
2961 
2962 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2963 			if (sz < 0) {
2964 				err = sz;
2965 				goto out;
2966 			}
2967 
2968 			cur += sz;
2969 		}
2970 	}
2971 
2972 	if (core_relos_sz) {
2973 		*(__u32 *)cur = core_relo_rec_sz;
2974 		cur += sizeof(__u32);
2975 
2976 		for (i = 1; i < linker->sec_cnt; i++) {
2977 			struct dst_sec *sec = &linker->secs[i];
2978 
2979 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2980 			if (sz < 0) {
2981 				err = sz;
2982 				goto out;
2983 			}
2984 
2985 			cur += sz;
2986 		}
2987 	}
2988 
2989 	linker->btf_ext = btf_ext__new(data, total_sz);
2990 	err = libbpf_get_error(linker->btf_ext);
2991 	if (err) {
2992 		linker->btf_ext = NULL;
2993 		pr_warn("failed to parse final .BTF.ext data: %s\n", errstr(err));
2994 		goto out;
2995 	}
2996 
2997 out:
2998 	free(data);
2999 	return err;
3000 }
3001