xref: /linux/tools/lib/bpf/linker.c (revision d0d106a2bd21499901299160744e5fe9f4c83ddb)
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2 /*
3  * BPF static linker
4  *
5  * Copyright (c) 2021 Facebook
6  */
7 #ifndef _GNU_SOURCE
8 #define _GNU_SOURCE
9 #endif
10 
11 #include <stdbool.h>
12 #include <stddef.h>
13 #include <stdio.h>
14 #include <stdlib.h>
15 #include <string.h>
16 #include <unistd.h>
17 #include <errno.h>
18 #include <linux/err.h>
19 #include <linux/btf.h>
20 #include <elf.h>
21 #include <libelf.h>
22 #include <fcntl.h>
23 #include <sys/mman.h>
24 #include "libbpf.h"
25 #include "btf.h"
26 #include "libbpf_internal.h"
27 #include "strset.h"
28 #include "str_error.h"
29 
30 #define BTF_EXTERN_SEC ".extern"
31 
32 struct src_sec {
33 	const char *sec_name;
34 	/* positional (not necessarily ELF) index in an array of sections */
35 	int id;
36 	/* positional (not necessarily ELF) index of a matching section in a final object file */
37 	int dst_id;
38 	/* section data offset in a matching output section */
39 	int dst_off;
40 	/* whether section is omitted from the final ELF file */
41 	bool skipped;
42 	/* whether section is an ephemeral section, not mapped to an ELF section */
43 	bool ephemeral;
44 
45 	/* ELF info */
46 	size_t sec_idx;
47 	Elf_Scn *scn;
48 	Elf64_Shdr *shdr;
49 	Elf_Data *data;
50 
51 	/* corresponding BTF DATASEC type ID */
52 	int sec_type_id;
53 };
54 
55 struct src_obj {
56 	const char *filename;
57 	int fd;
58 	Elf *elf;
59 	/* Section header strings section index */
60 	size_t shstrs_sec_idx;
61 	/* SYMTAB section index */
62 	size_t symtab_sec_idx;
63 
64 	struct btf *btf;
65 	struct btf_ext *btf_ext;
66 
67 	/* List of sections (including ephemeral). Slot zero is unused. */
68 	struct src_sec *secs;
69 	int sec_cnt;
70 
71 	/* mapping of symbol indices from src to dst ELF */
72 	int *sym_map;
73 	/* mapping from the src BTF type IDs to dst ones */
74 	int *btf_type_map;
75 };
76 
77 /* single .BTF.ext data section */
78 struct btf_ext_sec_data {
79 	size_t rec_cnt;
80 	__u32 rec_sz;
81 	void *recs;
82 };
83 
84 struct glob_sym {
85 	/* ELF symbol index */
86 	int sym_idx;
87 	/* associated section id for .ksyms, .kconfig, etc, but not .extern */
88 	int sec_id;
89 	/* extern name offset in STRTAB */
90 	int name_off;
91 	/* optional associated BTF type ID */
92 	int btf_id;
93 	/* BTF type ID to which VAR/FUNC type is pointing to; used for
94 	 * rewriting types when extern VAR/FUNC is resolved to a concrete
95 	 * definition
96 	 */
97 	int underlying_btf_id;
98 	/* sec_var index in the corresponding dst_sec, if exists */
99 	int var_idx;
100 
101 	/* extern or resolved/global symbol */
102 	bool is_extern;
103 	/* weak or strong symbol, never goes back from strong to weak */
104 	bool is_weak;
105 };
106 
107 struct dst_sec {
108 	char *sec_name;
109 	/* positional (not necessarily ELF) index in an array of sections */
110 	int id;
111 
112 	bool ephemeral;
113 
114 	/* ELF info */
115 	size_t sec_idx;
116 	Elf_Scn *scn;
117 	Elf64_Shdr *shdr;
118 	Elf_Data *data;
119 
120 	/* final output section size */
121 	int sec_sz;
122 	/* final output contents of the section */
123 	void *raw_data;
124 
125 	/* corresponding STT_SECTION symbol index in SYMTAB */
126 	int sec_sym_idx;
127 
128 	/* section's DATASEC variable info, emitted on BTF finalization */
129 	bool has_btf;
130 	int sec_var_cnt;
131 	struct btf_var_secinfo *sec_vars;
132 
133 	/* section's .BTF.ext data */
134 	struct btf_ext_sec_data func_info;
135 	struct btf_ext_sec_data line_info;
136 	struct btf_ext_sec_data core_relo_info;
137 };
138 
139 struct bpf_linker {
140 	char *filename;
141 	int fd;
142 	Elf *elf;
143 	Elf64_Ehdr *elf_hdr;
144 	bool swapped_endian;
145 
146 	/* Output sections metadata */
147 	struct dst_sec *secs;
148 	int sec_cnt;
149 
150 	struct strset *strtab_strs; /* STRTAB unique strings */
151 	size_t strtab_sec_idx; /* STRTAB section index */
152 	size_t symtab_sec_idx; /* SYMTAB section index */
153 
154 	struct btf *btf;
155 	struct btf_ext *btf_ext;
156 
157 	/* global (including extern) ELF symbols */
158 	int glob_sym_cnt;
159 	struct glob_sym *glob_syms;
160 
161 	bool fd_is_owned;
162 };
163 
164 #define pr_warn_elf(fmt, ...)									\
165 	libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
166 
167 static int init_output_elf(struct bpf_linker *linker);
168 
169 static int bpf_linker_add_file(struct bpf_linker *linker, int fd,
170 			       const char *filename);
171 
172 static int linker_load_obj_file(struct bpf_linker *linker,
173 				struct src_obj *obj);
174 static int linker_sanity_check_elf(struct src_obj *obj);
175 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
176 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
177 static int linker_sanity_check_btf(struct src_obj *obj);
178 static int linker_sanity_check_btf_ext(struct src_obj *obj);
179 static int linker_fixup_btf(struct src_obj *obj);
180 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
181 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
182 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
183 				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
184 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
185 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
186 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
187 
188 static int finalize_btf(struct bpf_linker *linker);
189 static int finalize_btf_ext(struct bpf_linker *linker);
190 
bpf_linker__free(struct bpf_linker * linker)191 void bpf_linker__free(struct bpf_linker *linker)
192 {
193 	int i;
194 
195 	if (!linker)
196 		return;
197 
198 	free(linker->filename);
199 
200 	if (linker->elf)
201 		elf_end(linker->elf);
202 
203 	if (linker->fd >= 0 && linker->fd_is_owned)
204 		close(linker->fd);
205 
206 	strset__free(linker->strtab_strs);
207 
208 	btf__free(linker->btf);
209 	btf_ext__free(linker->btf_ext);
210 
211 	for (i = 1; i < linker->sec_cnt; i++) {
212 		struct dst_sec *sec = &linker->secs[i];
213 
214 		free(sec->sec_name);
215 		free(sec->raw_data);
216 		free(sec->sec_vars);
217 
218 		free(sec->func_info.recs);
219 		free(sec->line_info.recs);
220 		free(sec->core_relo_info.recs);
221 	}
222 	free(linker->secs);
223 
224 	free(linker->glob_syms);
225 	free(linker);
226 }
227 
bpf_linker__new(const char * filename,struct bpf_linker_opts * opts)228 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
229 {
230 	struct bpf_linker *linker;
231 	int err;
232 
233 	if (!OPTS_VALID(opts, bpf_linker_opts))
234 		return errno = EINVAL, NULL;
235 
236 	if (elf_version(EV_CURRENT) == EV_NONE) {
237 		pr_warn_elf("libelf initialization failed");
238 		return errno = EINVAL, NULL;
239 	}
240 
241 	linker = calloc(1, sizeof(*linker));
242 	if (!linker)
243 		return errno = ENOMEM, NULL;
244 
245 	linker->filename = strdup(filename);
246 	if (!linker->filename) {
247 		err = -ENOMEM;
248 		goto err_out;
249 	}
250 
251 	linker->fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
252 	if (linker->fd < 0) {
253 		err = -errno;
254 		pr_warn("failed to create '%s': %d\n", filename, err);
255 		goto err_out;
256 	}
257 	linker->fd_is_owned = true;
258 
259 	err = init_output_elf(linker);
260 	if (err)
261 		goto err_out;
262 
263 	return linker;
264 
265 err_out:
266 	bpf_linker__free(linker);
267 	return errno = -err, NULL;
268 }
269 
bpf_linker__new_fd(int fd,struct bpf_linker_opts * opts)270 struct bpf_linker *bpf_linker__new_fd(int fd, struct bpf_linker_opts *opts)
271 {
272 	struct bpf_linker *linker;
273 	char filename[32];
274 	int err;
275 
276 	if (fd < 0)
277 		return errno = EINVAL, NULL;
278 
279 	if (!OPTS_VALID(opts, bpf_linker_opts))
280 		return errno = EINVAL, NULL;
281 
282 	if (elf_version(EV_CURRENT) == EV_NONE) {
283 		pr_warn_elf("libelf initialization failed");
284 		return errno = EINVAL, NULL;
285 	}
286 
287 	linker = calloc(1, sizeof(*linker));
288 	if (!linker)
289 		return errno = ENOMEM, NULL;
290 
291 	snprintf(filename, sizeof(filename), "fd:%d", fd);
292 	linker->filename = strdup(filename);
293 	if (!linker->filename) {
294 		err = -ENOMEM;
295 		goto err_out;
296 	}
297 
298 	linker->fd = fd;
299 	linker->fd_is_owned = false;
300 
301 	err = init_output_elf(linker);
302 	if (err)
303 		goto err_out;
304 
305 	return linker;
306 
307 err_out:
308 	bpf_linker__free(linker);
309 	return errno = -err, NULL;
310 }
311 
add_dst_sec(struct bpf_linker * linker,const char * sec_name)312 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
313 {
314 	struct dst_sec *secs = linker->secs, *sec;
315 	size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
316 
317 	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
318 	if (!secs)
319 		return NULL;
320 
321 	/* zero out newly allocated memory */
322 	memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
323 
324 	linker->secs = secs;
325 	linker->sec_cnt = new_cnt;
326 
327 	sec = &linker->secs[new_cnt - 1];
328 	sec->id = new_cnt - 1;
329 	sec->sec_name = strdup(sec_name);
330 	if (!sec->sec_name)
331 		return NULL;
332 
333 	return sec;
334 }
335 
add_new_sym(struct bpf_linker * linker,size_t * sym_idx)336 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
337 {
338 	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
339 	Elf64_Sym *syms, *sym;
340 	size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
341 
342 	syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
343 	if (!syms)
344 		return NULL;
345 
346 	sym = &syms[sym_cnt];
347 	memset(sym, 0, sizeof(*sym));
348 
349 	symtab->raw_data = syms;
350 	symtab->sec_sz += sizeof(*sym);
351 	symtab->shdr->sh_size += sizeof(*sym);
352 	symtab->data->d_size += sizeof(*sym);
353 
354 	if (sym_idx)
355 		*sym_idx = sym_cnt;
356 
357 	return sym;
358 }
359 
init_output_elf(struct bpf_linker * linker)360 static int init_output_elf(struct bpf_linker *linker)
361 {
362 	int err, str_off;
363 	Elf64_Sym *init_sym;
364 	struct dst_sec *sec;
365 
366 	linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
367 	if (!linker->elf) {
368 		pr_warn_elf("failed to create ELF object");
369 		return -EINVAL;
370 	}
371 
372 	/* ELF header */
373 	linker->elf_hdr = elf64_newehdr(linker->elf);
374 	if (!linker->elf_hdr) {
375 		pr_warn_elf("failed to create ELF header");
376 		return -EINVAL;
377 	}
378 
379 	linker->elf_hdr->e_machine = EM_BPF;
380 	linker->elf_hdr->e_type = ET_REL;
381 	/* Set unknown ELF endianness, assign later from input files */
382 	linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE;
383 
384 	/* STRTAB */
385 	/* initialize strset with an empty string to conform to ELF */
386 	linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
387 	if (libbpf_get_error(linker->strtab_strs))
388 		return libbpf_get_error(linker->strtab_strs);
389 
390 	sec = add_dst_sec(linker, ".strtab");
391 	if (!sec)
392 		return -ENOMEM;
393 
394 	sec->scn = elf_newscn(linker->elf);
395 	if (!sec->scn) {
396 		pr_warn_elf("failed to create STRTAB section");
397 		return -EINVAL;
398 	}
399 
400 	sec->shdr = elf64_getshdr(sec->scn);
401 	if (!sec->shdr)
402 		return -EINVAL;
403 
404 	sec->data = elf_newdata(sec->scn);
405 	if (!sec->data) {
406 		pr_warn_elf("failed to create STRTAB data");
407 		return -EINVAL;
408 	}
409 
410 	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
411 	if (str_off < 0)
412 		return str_off;
413 
414 	sec->sec_idx = elf_ndxscn(sec->scn);
415 	linker->elf_hdr->e_shstrndx = sec->sec_idx;
416 	linker->strtab_sec_idx = sec->sec_idx;
417 
418 	sec->shdr->sh_name = str_off;
419 	sec->shdr->sh_type = SHT_STRTAB;
420 	sec->shdr->sh_flags = SHF_STRINGS;
421 	sec->shdr->sh_offset = 0;
422 	sec->shdr->sh_link = 0;
423 	sec->shdr->sh_info = 0;
424 	sec->shdr->sh_addralign = 1;
425 	sec->shdr->sh_size = sec->sec_sz = 0;
426 	sec->shdr->sh_entsize = 0;
427 
428 	/* SYMTAB */
429 	sec = add_dst_sec(linker, ".symtab");
430 	if (!sec)
431 		return -ENOMEM;
432 
433 	sec->scn = elf_newscn(linker->elf);
434 	if (!sec->scn) {
435 		pr_warn_elf("failed to create SYMTAB section");
436 		return -EINVAL;
437 	}
438 
439 	sec->shdr = elf64_getshdr(sec->scn);
440 	if (!sec->shdr)
441 		return -EINVAL;
442 
443 	sec->data = elf_newdata(sec->scn);
444 	if (!sec->data) {
445 		pr_warn_elf("failed to create SYMTAB data");
446 		return -EINVAL;
447 	}
448 	/* Ensure libelf translates byte-order of symbol records */
449 	sec->data->d_type = ELF_T_SYM;
450 
451 	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
452 	if (str_off < 0)
453 		return str_off;
454 
455 	sec->sec_idx = elf_ndxscn(sec->scn);
456 	linker->symtab_sec_idx = sec->sec_idx;
457 
458 	sec->shdr->sh_name = str_off;
459 	sec->shdr->sh_type = SHT_SYMTAB;
460 	sec->shdr->sh_flags = 0;
461 	sec->shdr->sh_offset = 0;
462 	sec->shdr->sh_link = linker->strtab_sec_idx;
463 	/* sh_info should be one greater than the index of the last local
464 	 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
465 	 */
466 	sec->shdr->sh_info = 0;
467 	sec->shdr->sh_addralign = 8;
468 	sec->shdr->sh_entsize = sizeof(Elf64_Sym);
469 
470 	/* .BTF */
471 	linker->btf = btf__new_empty();
472 	err = libbpf_get_error(linker->btf);
473 	if (err)
474 		return err;
475 
476 	/* add the special all-zero symbol */
477 	init_sym = add_new_sym(linker, NULL);
478 	if (!init_sym)
479 		return -EINVAL;
480 
481 	init_sym->st_name = 0;
482 	init_sym->st_info = 0;
483 	init_sym->st_other = 0;
484 	init_sym->st_shndx = SHN_UNDEF;
485 	init_sym->st_value = 0;
486 	init_sym->st_size = 0;
487 
488 	return 0;
489 }
490 
bpf_linker_add_file(struct bpf_linker * linker,int fd,const char * filename)491 static int bpf_linker_add_file(struct bpf_linker *linker, int fd,
492 			       const char *filename)
493 {
494 	struct src_obj obj = {};
495 	int err = 0;
496 
497 	obj.filename = filename;
498 	obj.fd = fd;
499 
500 	err = err ?: linker_load_obj_file(linker, &obj);
501 	err = err ?: linker_append_sec_data(linker, &obj);
502 	err = err ?: linker_append_elf_syms(linker, &obj);
503 	err = err ?: linker_append_elf_relos(linker, &obj);
504 	err = err ?: linker_append_btf(linker, &obj);
505 	err = err ?: linker_append_btf_ext(linker, &obj);
506 
507 	/* free up src_obj resources */
508 	free(obj.btf_type_map);
509 	btf__free(obj.btf);
510 	btf_ext__free(obj.btf_ext);
511 	free(obj.secs);
512 	free(obj.sym_map);
513 	if (obj.elf)
514 		elf_end(obj.elf);
515 
516 	return err;
517 }
518 
bpf_linker__add_file(struct bpf_linker * linker,const char * filename,const struct bpf_linker_file_opts * opts)519 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
520 			 const struct bpf_linker_file_opts *opts)
521 {
522 	int fd, err;
523 
524 	if (!OPTS_VALID(opts, bpf_linker_file_opts))
525 		return libbpf_err(-EINVAL);
526 
527 	if (!linker->elf)
528 		return libbpf_err(-EINVAL);
529 
530 	fd = open(filename, O_RDONLY | O_CLOEXEC);
531 	if (fd < 0) {
532 		err = -errno;
533 		pr_warn("failed to open file '%s': %s\n", filename, errstr(err));
534 		return libbpf_err(err);
535 	}
536 
537 	err = bpf_linker_add_file(linker, fd, filename);
538 	close(fd);
539 	return libbpf_err(err);
540 }
541 
bpf_linker__add_fd(struct bpf_linker * linker,int fd,const struct bpf_linker_file_opts * opts)542 int bpf_linker__add_fd(struct bpf_linker *linker, int fd,
543 		       const struct bpf_linker_file_opts *opts)
544 {
545 	char filename[32];
546 	int err;
547 
548 	if (!OPTS_VALID(opts, bpf_linker_file_opts))
549 		return libbpf_err(-EINVAL);
550 
551 	if (!linker->elf)
552 		return libbpf_err(-EINVAL);
553 
554 	if (fd < 0)
555 		return libbpf_err(-EINVAL);
556 
557 	snprintf(filename, sizeof(filename), "fd:%d", fd);
558 	err = bpf_linker_add_file(linker, fd, filename);
559 	return libbpf_err(err);
560 }
561 
bpf_linker__add_buf(struct bpf_linker * linker,void * buf,size_t buf_sz,const struct bpf_linker_file_opts * opts)562 int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz,
563 			const struct bpf_linker_file_opts *opts)
564 {
565 	char filename[32];
566 	int fd, written, ret;
567 
568 	if (!OPTS_VALID(opts, bpf_linker_file_opts))
569 		return libbpf_err(-EINVAL);
570 
571 	if (!linker->elf)
572 		return libbpf_err(-EINVAL);
573 
574 	snprintf(filename, sizeof(filename), "mem:%p+%zu", buf, buf_sz);
575 
576 	fd = memfd_create(filename, 0);
577 	if (fd < 0) {
578 		ret = -errno;
579 		pr_warn("failed to create memfd '%s': %s\n", filename, errstr(ret));
580 		return libbpf_err(ret);
581 	}
582 
583 	written = 0;
584 	while (written < buf_sz) {
585 		ret = write(fd, buf, buf_sz);
586 		if (ret < 0) {
587 			ret = -errno;
588 			pr_warn("failed to write '%s': %s\n", filename, errstr(ret));
589 			goto err_out;
590 		}
591 		written += ret;
592 	}
593 
594 	ret = bpf_linker_add_file(linker, fd, filename);
595 err_out:
596 	close(fd);
597 	return libbpf_err(ret);
598 }
599 
is_dwarf_sec_name(const char * name)600 static bool is_dwarf_sec_name(const char *name)
601 {
602 	/* approximation, but the actual list is too long */
603 	return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
604 }
605 
is_ignored_sec(struct src_sec * sec)606 static bool is_ignored_sec(struct src_sec *sec)
607 {
608 	Elf64_Shdr *shdr = sec->shdr;
609 	const char *name = sec->sec_name;
610 
611 	/* no special handling of .strtab */
612 	if (shdr->sh_type == SHT_STRTAB)
613 		return true;
614 
615 	/* ignore .llvm_addrsig section as well */
616 	if (shdr->sh_type == SHT_LLVM_ADDRSIG)
617 		return true;
618 
619 	/* no subprograms will lead to an empty .text section, ignore it */
620 	if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
621 	    strcmp(sec->sec_name, ".text") == 0)
622 		return true;
623 
624 	/* DWARF sections */
625 	if (is_dwarf_sec_name(sec->sec_name))
626 		return true;
627 
628 	if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
629 		name += sizeof(".rel") - 1;
630 		/* DWARF section relocations */
631 		if (is_dwarf_sec_name(name))
632 			return true;
633 
634 		/* .BTF and .BTF.ext don't need relocations */
635 		if (strcmp(name, BTF_ELF_SEC) == 0 ||
636 		    strcmp(name, BTF_EXT_ELF_SEC) == 0)
637 			return true;
638 	}
639 
640 	return false;
641 }
642 
add_src_sec(struct src_obj * obj,const char * sec_name)643 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
644 {
645 	struct src_sec *secs = obj->secs, *sec;
646 	size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
647 
648 	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
649 	if (!secs)
650 		return NULL;
651 
652 	/* zero out newly allocated memory */
653 	memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
654 
655 	obj->secs = secs;
656 	obj->sec_cnt = new_cnt;
657 
658 	sec = &obj->secs[new_cnt - 1];
659 	sec->id = new_cnt - 1;
660 	sec->sec_name = sec_name;
661 
662 	return sec;
663 }
664 
linker_load_obj_file(struct bpf_linker * linker,struct src_obj * obj)665 static int linker_load_obj_file(struct bpf_linker *linker,
666 				struct src_obj *obj)
667 {
668 	int err = 0;
669 	Elf_Scn *scn;
670 	Elf_Data *data;
671 	Elf64_Ehdr *ehdr;
672 	Elf64_Shdr *shdr;
673 	struct src_sec *sec;
674 	unsigned char obj_byteorder;
675 	unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA];
676 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
677 	const unsigned char host_byteorder = ELFDATA2LSB;
678 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
679 	const unsigned char host_byteorder = ELFDATA2MSB;
680 #else
681 #error "Unknown __BYTE_ORDER__"
682 #endif
683 
684 	pr_debug("linker: adding object file '%s'...\n", obj->filename);
685 
686 	obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
687 	if (!obj->elf) {
688 		pr_warn_elf("failed to parse ELF file '%s'", obj->filename);
689 		return -EINVAL;
690 	}
691 
692 	/* Sanity check ELF file high-level properties */
693 	ehdr = elf64_getehdr(obj->elf);
694 	if (!ehdr) {
695 		pr_warn_elf("failed to get ELF header for %s", obj->filename);
696 		return -EINVAL;
697 	}
698 
699 	/* Linker output endianness set by first input object */
700 	obj_byteorder = ehdr->e_ident[EI_DATA];
701 	if (obj_byteorder != ELFDATA2LSB && obj_byteorder != ELFDATA2MSB) {
702 		err = -EOPNOTSUPP;
703 		pr_warn("unknown byte order of ELF file %s\n", obj->filename);
704 		return err;
705 	}
706 	if (link_byteorder == ELFDATANONE) {
707 		linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder;
708 		linker->swapped_endian = obj_byteorder != host_byteorder;
709 		pr_debug("linker: set %s-endian output byte order\n",
710 			 obj_byteorder == ELFDATA2MSB ? "big" : "little");
711 	} else if (link_byteorder != obj_byteorder) {
712 		err = -EOPNOTSUPP;
713 		pr_warn("byte order mismatch with ELF file %s\n", obj->filename);
714 		return err;
715 	}
716 
717 	if (ehdr->e_type != ET_REL
718 	    || ehdr->e_machine != EM_BPF
719 	    || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
720 		err = -EOPNOTSUPP;
721 		pr_warn_elf("unsupported kind of ELF file %s", obj->filename);
722 		return err;
723 	}
724 
725 	if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
726 		pr_warn_elf("failed to get SHSTRTAB section index for %s", obj->filename);
727 		return -EINVAL;
728 	}
729 
730 	scn = NULL;
731 	while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
732 		size_t sec_idx = elf_ndxscn(scn);
733 		const char *sec_name;
734 
735 		shdr = elf64_getshdr(scn);
736 		if (!shdr) {
737 			pr_warn_elf("failed to get section #%zu header for %s",
738 				    sec_idx, obj->filename);
739 			return -EINVAL;
740 		}
741 
742 		sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
743 		if (!sec_name) {
744 			pr_warn_elf("failed to get section #%zu name for %s",
745 				    sec_idx, obj->filename);
746 			return -EINVAL;
747 		}
748 
749 		data = elf_getdata(scn, 0);
750 		if (!data) {
751 			pr_warn_elf("failed to get section #%zu (%s) data from %s",
752 				    sec_idx, sec_name, obj->filename);
753 			return -EINVAL;
754 		}
755 
756 		sec = add_src_sec(obj, sec_name);
757 		if (!sec)
758 			return -ENOMEM;
759 
760 		sec->scn = scn;
761 		sec->shdr = shdr;
762 		sec->data = data;
763 		sec->sec_idx = elf_ndxscn(scn);
764 
765 		if (is_ignored_sec(sec)) {
766 			sec->skipped = true;
767 			continue;
768 		}
769 
770 		switch (shdr->sh_type) {
771 		case SHT_SYMTAB:
772 			if (obj->symtab_sec_idx) {
773 				err = -EOPNOTSUPP;
774 				pr_warn("multiple SYMTAB sections found, not supported\n");
775 				return err;
776 			}
777 			obj->symtab_sec_idx = sec_idx;
778 			break;
779 		case SHT_STRTAB:
780 			/* we'll construct our own string table */
781 			break;
782 		case SHT_PROGBITS:
783 			if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
784 				obj->btf = btf__new(data->d_buf, shdr->sh_size);
785 				err = libbpf_get_error(obj->btf);
786 				if (err) {
787 					pr_warn("failed to parse .BTF from %s: %s\n",
788 						obj->filename, errstr(err));
789 					return err;
790 				}
791 				sec->skipped = true;
792 				continue;
793 			}
794 			if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
795 				obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
796 				err = libbpf_get_error(obj->btf_ext);
797 				if (err) {
798 					pr_warn("failed to parse .BTF.ext from '%s': %s\n",
799 						obj->filename, errstr(err));
800 					return err;
801 				}
802 				sec->skipped = true;
803 				continue;
804 			}
805 
806 			/* data & code */
807 			break;
808 		case SHT_NOBITS:
809 			/* BSS */
810 			break;
811 		case SHT_REL:
812 			/* relocations */
813 			break;
814 		default:
815 			pr_warn("unrecognized section #%zu (%s) in %s\n",
816 				sec_idx, sec_name, obj->filename);
817 			err = -EINVAL;
818 			return err;
819 		}
820 	}
821 
822 	err = err ?: linker_sanity_check_elf(obj);
823 	err = err ?: linker_sanity_check_btf(obj);
824 	err = err ?: linker_sanity_check_btf_ext(obj);
825 	err = err ?: linker_fixup_btf(obj);
826 
827 	return err;
828 }
829 
linker_sanity_check_elf(struct src_obj * obj)830 static int linker_sanity_check_elf(struct src_obj *obj)
831 {
832 	struct src_sec *sec;
833 	int i, err;
834 
835 	if (!obj->symtab_sec_idx) {
836 		pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
837 		return -EINVAL;
838 	}
839 	if (!obj->shstrs_sec_idx) {
840 		pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
841 		return -EINVAL;
842 	}
843 
844 	for (i = 1; i < obj->sec_cnt; i++) {
845 		sec = &obj->secs[i];
846 
847 		if (sec->sec_name[0] == '\0') {
848 			pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
849 			return -EINVAL;
850 		}
851 
852 		if (is_dwarf_sec_name(sec->sec_name))
853 			continue;
854 
855 		if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {
856 			pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",
857 				sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
858 				obj->filename);
859 			return -EINVAL;
860 		}
861 		if (sec->shdr->sh_addralign != sec->data->d_align) {
862 			pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",
863 				sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
864 				(long long unsigned)sec->data->d_align, obj->filename);
865 			return -EINVAL;
866 		}
867 
868 		if (sec->shdr->sh_size != sec->data->d_size) {
869 			pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",
870 				sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
871 				(long long unsigned)sec->data->d_size, obj->filename);
872 			return -EINVAL;
873 		}
874 
875 		switch (sec->shdr->sh_type) {
876 		case SHT_SYMTAB:
877 			err = linker_sanity_check_elf_symtab(obj, sec);
878 			if (err)
879 				return err;
880 			break;
881 		case SHT_STRTAB:
882 			break;
883 		case SHT_PROGBITS:
884 			if (sec->shdr->sh_flags & SHF_EXECINSTR) {
885 				if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {
886 					pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",
887 						sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
888 						obj->filename);
889 					return -EINVAL;
890 				}
891 			}
892 			break;
893 		case SHT_NOBITS:
894 			break;
895 		case SHT_REL:
896 			err = linker_sanity_check_elf_relos(obj, sec);
897 			if (err)
898 				return err;
899 			break;
900 		case SHT_LLVM_ADDRSIG:
901 			break;
902 		default:
903 			pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
904 				sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
905 			return -EINVAL;
906 		}
907 	}
908 
909 	return 0;
910 }
911 
linker_sanity_check_elf_symtab(struct src_obj * obj,struct src_sec * sec)912 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
913 {
914 	struct src_sec *link_sec;
915 	Elf64_Sym *sym;
916 	int i, n;
917 
918 	if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
919 		return -EINVAL;
920 	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
921 		return -EINVAL;
922 
923 	if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
924 		pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
925 			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
926 		return -EINVAL;
927 	}
928 	link_sec = &obj->secs[sec->shdr->sh_link];
929 	if (link_sec->shdr->sh_type != SHT_STRTAB) {
930 		pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
931 			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
932 		return -EINVAL;
933 	}
934 
935 	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
936 	sym = sec->data->d_buf;
937 	for (i = 0; i < n; i++, sym++) {
938 		int sym_type = ELF64_ST_TYPE(sym->st_info);
939 		int sym_bind = ELF64_ST_BIND(sym->st_info);
940 		int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
941 
942 		if (i == 0) {
943 			if (sym->st_name != 0 || sym->st_info != 0
944 			    || sym->st_other != 0 || sym->st_shndx != 0
945 			    || sym->st_value != 0 || sym->st_size != 0) {
946 				pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
947 				return -EINVAL;
948 			}
949 			continue;
950 		}
951 		if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
952 			pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
953 				i, sec->sec_idx, sym_bind);
954 			return -EINVAL;
955 		}
956 		if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
957 			pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
958 				i, sec->sec_idx, sym_vis);
959 			return -EINVAL;
960 		}
961 		if (sym->st_shndx == 0) {
962 			if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
963 			    || sym->st_value != 0 || sym->st_size != 0) {
964 				pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
965 					i, obj->filename);
966 
967 				return -EINVAL;
968 			}
969 			continue;
970 		}
971 		if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
972 			pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
973 				i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
974 			return -EINVAL;
975 		}
976 		if (sym_type == STT_SECTION) {
977 			if (sym->st_value != 0)
978 				return -EINVAL;
979 			continue;
980 		}
981 	}
982 
983 	return 0;
984 }
985 
linker_sanity_check_elf_relos(struct src_obj * obj,struct src_sec * sec)986 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
987 {
988 	struct src_sec *link_sec, *sym_sec;
989 	Elf64_Rel *relo;
990 	int i, n;
991 
992 	if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
993 		return -EINVAL;
994 	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
995 		return -EINVAL;
996 
997 	/* SHT_REL's sh_link should point to SYMTAB */
998 	if (sec->shdr->sh_link != obj->symtab_sec_idx) {
999 		pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
1000 			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
1001 		return -EINVAL;
1002 	}
1003 
1004 	/* SHT_REL's sh_info points to relocated section */
1005 	if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
1006 		pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
1007 			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
1008 		return -EINVAL;
1009 	}
1010 	link_sec = &obj->secs[sec->shdr->sh_info];
1011 
1012 	/* .rel<secname> -> <secname> pattern is followed */
1013 	if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
1014 	    || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
1015 		pr_warn("ELF relo section #%zu name has invalid name in %s\n",
1016 			sec->sec_idx, obj->filename);
1017 		return -EINVAL;
1018 	}
1019 
1020 	/* don't further validate relocations for ignored sections */
1021 	if (link_sec->skipped)
1022 		return 0;
1023 
1024 	/* relocatable section is data or instructions */
1025 	if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
1026 		pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
1027 			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
1028 		return -EINVAL;
1029 	}
1030 
1031 	/* check sanity of each relocation */
1032 	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
1033 	relo = sec->data->d_buf;
1034 	sym_sec = &obj->secs[obj->symtab_sec_idx];
1035 	for (i = 0; i < n; i++, relo++) {
1036 		size_t sym_idx = ELF64_R_SYM(relo->r_info);
1037 		size_t sym_type = ELF64_R_TYPE(relo->r_info);
1038 
1039 		if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
1040 		    sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
1041 			pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
1042 				i, sec->sec_idx, sym_type, obj->filename);
1043 			return -EINVAL;
1044 		}
1045 
1046 		if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
1047 			pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
1048 				i, sec->sec_idx, sym_idx, obj->filename);
1049 			return -EINVAL;
1050 		}
1051 
1052 		if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
1053 			if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
1054 				pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
1055 					i, sec->sec_idx, sym_idx, obj->filename);
1056 				return -EINVAL;
1057 			}
1058 		}
1059 	}
1060 
1061 	return 0;
1062 }
1063 
check_btf_type_id(__u32 * type_id,void * ctx)1064 static int check_btf_type_id(__u32 *type_id, void *ctx)
1065 {
1066 	struct btf *btf = ctx;
1067 
1068 	if (*type_id >= btf__type_cnt(btf))
1069 		return -EINVAL;
1070 
1071 	return 0;
1072 }
1073 
check_btf_str_off(__u32 * str_off,void * ctx)1074 static int check_btf_str_off(__u32 *str_off, void *ctx)
1075 {
1076 	struct btf *btf = ctx;
1077 	const char *s;
1078 
1079 	s = btf__str_by_offset(btf, *str_off);
1080 
1081 	if (!s)
1082 		return -EINVAL;
1083 
1084 	return 0;
1085 }
1086 
linker_sanity_check_btf(struct src_obj * obj)1087 static int linker_sanity_check_btf(struct src_obj *obj)
1088 {
1089 	struct btf_type *t;
1090 	int i, n, err;
1091 
1092 	if (!obj->btf)
1093 		return 0;
1094 
1095 	n = btf__type_cnt(obj->btf);
1096 	for (i = 1; i < n; i++) {
1097 		struct btf_field_iter it;
1098 		__u32 *type_id, *str_off;
1099 
1100 		t = btf_type_by_id(obj->btf, i);
1101 
1102 		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
1103 		if (err)
1104 			return err;
1105 		while ((type_id = btf_field_iter_next(&it))) {
1106 			if (*type_id >= n)
1107 				return -EINVAL;
1108 		}
1109 
1110 		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
1111 		if (err)
1112 			return err;
1113 		while ((str_off = btf_field_iter_next(&it))) {
1114 			if (!btf__str_by_offset(obj->btf, *str_off))
1115 				return -EINVAL;
1116 		}
1117 	}
1118 
1119 	return 0;
1120 }
1121 
linker_sanity_check_btf_ext(struct src_obj * obj)1122 static int linker_sanity_check_btf_ext(struct src_obj *obj)
1123 {
1124 	int err = 0;
1125 
1126 	if (!obj->btf_ext)
1127 		return 0;
1128 
1129 	/* can't use .BTF.ext without .BTF */
1130 	if (!obj->btf)
1131 		return -EINVAL;
1132 
1133 	err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
1134 	err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
1135 	if (err)
1136 		return err;
1137 
1138 	return 0;
1139 }
1140 
init_sec(struct bpf_linker * linker,struct dst_sec * dst_sec,struct src_sec * src_sec)1141 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
1142 {
1143 	Elf_Scn *scn;
1144 	Elf_Data *data;
1145 	Elf64_Shdr *shdr;
1146 	int name_off;
1147 
1148 	dst_sec->sec_sz = 0;
1149 	dst_sec->sec_idx = 0;
1150 	dst_sec->ephemeral = src_sec->ephemeral;
1151 
1152 	/* ephemeral sections are just thin section shells lacking most parts */
1153 	if (src_sec->ephemeral)
1154 		return 0;
1155 
1156 	scn = elf_newscn(linker->elf);
1157 	if (!scn)
1158 		return -ENOMEM;
1159 	data = elf_newdata(scn);
1160 	if (!data)
1161 		return -ENOMEM;
1162 	shdr = elf64_getshdr(scn);
1163 	if (!shdr)
1164 		return -ENOMEM;
1165 
1166 	dst_sec->scn = scn;
1167 	dst_sec->shdr = shdr;
1168 	dst_sec->data = data;
1169 	dst_sec->sec_idx = elf_ndxscn(scn);
1170 
1171 	name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1172 	if (name_off < 0)
1173 		return name_off;
1174 
1175 	shdr->sh_name = name_off;
1176 	shdr->sh_type = src_sec->shdr->sh_type;
1177 	shdr->sh_flags = src_sec->shdr->sh_flags;
1178 	shdr->sh_size = 0;
1179 	/* sh_link and sh_info have different meaning for different types of
1180 	 * sections, so we leave it up to the caller code to fill them in, if
1181 	 * necessary
1182 	 */
1183 	shdr->sh_link = 0;
1184 	shdr->sh_info = 0;
1185 	shdr->sh_addralign = src_sec->shdr->sh_addralign;
1186 	shdr->sh_entsize = src_sec->shdr->sh_entsize;
1187 
1188 	data->d_type = src_sec->data->d_type;
1189 	data->d_size = 0;
1190 	data->d_buf = NULL;
1191 	data->d_align = src_sec->data->d_align;
1192 	data->d_off = 0;
1193 
1194 	return 0;
1195 }
1196 
find_dst_sec_by_name(struct bpf_linker * linker,const char * sec_name)1197 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1198 {
1199 	struct dst_sec *sec;
1200 	int i;
1201 
1202 	for (i = 1; i < linker->sec_cnt; i++) {
1203 		sec = &linker->secs[i];
1204 
1205 		if (strcmp(sec->sec_name, sec_name) == 0)
1206 			return sec;
1207 	}
1208 
1209 	return NULL;
1210 }
1211 
secs_match(struct dst_sec * dst,struct src_sec * src)1212 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1213 {
1214 	if (dst->ephemeral || src->ephemeral)
1215 		return true;
1216 
1217 	if (dst->shdr->sh_type != src->shdr->sh_type) {
1218 		pr_warn("sec %s types mismatch\n", dst->sec_name);
1219 		return false;
1220 	}
1221 	if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1222 		pr_warn("sec %s flags mismatch\n", dst->sec_name);
1223 		return false;
1224 	}
1225 	if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1226 		pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1227 		return false;
1228 	}
1229 
1230 	return true;
1231 }
1232 
sec_content_is_same(struct dst_sec * dst_sec,struct src_sec * src_sec)1233 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1234 {
1235 	if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1236 		return false;
1237 	if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1238 		return false;
1239 	return true;
1240 }
1241 
is_exec_sec(struct dst_sec * sec)1242 static bool is_exec_sec(struct dst_sec *sec)
1243 {
1244 	if (!sec || sec->ephemeral)
1245 		return false;
1246 	return (sec->shdr->sh_type == SHT_PROGBITS) &&
1247 	       (sec->shdr->sh_flags & SHF_EXECINSTR);
1248 }
1249 
exec_sec_bswap(void * raw_data,int size)1250 static void exec_sec_bswap(void *raw_data, int size)
1251 {
1252 	const int insn_cnt = size / sizeof(struct bpf_insn);
1253 	struct bpf_insn *insn = raw_data;
1254 	int i;
1255 
1256 	for (i = 0; i < insn_cnt; i++, insn++)
1257 		bpf_insn_bswap(insn);
1258 }
1259 
extend_sec(struct bpf_linker * linker,struct dst_sec * dst,struct src_sec * src)1260 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1261 {
1262 	void *tmp;
1263 	size_t dst_align, src_align;
1264 	size_t dst_align_sz, dst_final_sz;
1265 	int err;
1266 
1267 	/* Ephemeral source section doesn't contribute anything to ELF
1268 	 * section data.
1269 	 */
1270 	if (src->ephemeral)
1271 		return 0;
1272 
1273 	/* Some sections (like .maps) can contain both externs (and thus be
1274 	 * ephemeral) and non-externs (map definitions). So it's possible that
1275 	 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1276 	 * first non-ephemeral entity appears. In such case, we add ELF
1277 	 * section, data, etc.
1278 	 */
1279 	if (dst->ephemeral) {
1280 		err = init_sec(linker, dst, src);
1281 		if (err)
1282 			return err;
1283 	}
1284 
1285 	dst_align = dst->shdr->sh_addralign;
1286 	src_align = src->shdr->sh_addralign;
1287 	if (dst_align == 0)
1288 		dst_align = 1;
1289 	if (dst_align < src_align)
1290 		dst_align = src_align;
1291 
1292 	dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1293 
1294 	/* no need to re-align final size */
1295 	dst_final_sz = dst_align_sz + src->shdr->sh_size;
1296 
1297 	if (src->shdr->sh_type != SHT_NOBITS) {
1298 		tmp = realloc(dst->raw_data, dst_final_sz);
1299 		/* If dst_align_sz == 0, realloc() behaves in a special way:
1300 		 * 1. When dst->raw_data is NULL it returns:
1301 		 *    "either NULL or a pointer suitable to be passed to free()" [1].
1302 		 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1303 		 *    thus invalidating any "pointer suitable to be passed to free()" obtained
1304 		 *    at step (1).
1305 		 *
1306 		 * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1307 		 * dst->raw_data would be freed again in bpf_linker__free().
1308 		 *
1309 		 * [1] man 3 realloc
1310 		 */
1311 		if (!tmp && dst_align_sz > 0)
1312 			return -ENOMEM;
1313 		dst->raw_data = tmp;
1314 
1315 		/* pad dst section, if it's alignment forced size increase */
1316 		memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1317 		/* now copy src data at a properly aligned offset */
1318 		memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1319 
1320 		/* convert added bpf insns to native byte-order */
1321 		if (linker->swapped_endian && is_exec_sec(dst))
1322 			exec_sec_bswap(dst->raw_data + dst_align_sz, src->shdr->sh_size);
1323 	}
1324 
1325 	dst->sec_sz = dst_final_sz;
1326 	dst->shdr->sh_size = dst_final_sz;
1327 	dst->data->d_size = dst_final_sz;
1328 
1329 	dst->shdr->sh_addralign = dst_align;
1330 	dst->data->d_align = dst_align;
1331 
1332 	src->dst_off = dst_align_sz;
1333 
1334 	return 0;
1335 }
1336 
is_data_sec(struct src_sec * sec)1337 static bool is_data_sec(struct src_sec *sec)
1338 {
1339 	if (!sec || sec->skipped)
1340 		return false;
1341 	/* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1342 	if (sec->ephemeral)
1343 		return true;
1344 	return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1345 }
1346 
is_relo_sec(struct src_sec * sec)1347 static bool is_relo_sec(struct src_sec *sec)
1348 {
1349 	if (!sec || sec->skipped || sec->ephemeral)
1350 		return false;
1351 	return sec->shdr->sh_type == SHT_REL;
1352 }
1353 
linker_append_sec_data(struct bpf_linker * linker,struct src_obj * obj)1354 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1355 {
1356 	int i, err;
1357 
1358 	for (i = 1; i < obj->sec_cnt; i++) {
1359 		struct src_sec *src_sec;
1360 		struct dst_sec *dst_sec;
1361 
1362 		src_sec = &obj->secs[i];
1363 		if (!is_data_sec(src_sec))
1364 			continue;
1365 
1366 		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1367 		if (!dst_sec) {
1368 			dst_sec = add_dst_sec(linker, src_sec->sec_name);
1369 			if (!dst_sec)
1370 				return -ENOMEM;
1371 			err = init_sec(linker, dst_sec, src_sec);
1372 			if (err) {
1373 				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1374 				return err;
1375 			}
1376 		} else {
1377 			if (!secs_match(dst_sec, src_sec)) {
1378 				pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1379 				return -1;
1380 			}
1381 
1382 			/* "license" and "version" sections are deduped */
1383 			if (strcmp(src_sec->sec_name, "license") == 0
1384 			    || strcmp(src_sec->sec_name, "version") == 0) {
1385 				if (!sec_content_is_same(dst_sec, src_sec)) {
1386 					pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1387 					return -EINVAL;
1388 				}
1389 				src_sec->skipped = true;
1390 				src_sec->dst_id = dst_sec->id;
1391 				continue;
1392 			}
1393 		}
1394 
1395 		/* record mapped section index */
1396 		src_sec->dst_id = dst_sec->id;
1397 
1398 		err = extend_sec(linker, dst_sec, src_sec);
1399 		if (err)
1400 			return err;
1401 	}
1402 
1403 	return 0;
1404 }
1405 
linker_append_elf_syms(struct bpf_linker * linker,struct src_obj * obj)1406 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1407 {
1408 	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1409 	Elf64_Sym *sym = symtab->data->d_buf;
1410 	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1411 	int str_sec_idx = symtab->shdr->sh_link;
1412 	const char *sym_name;
1413 
1414 	obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1415 	if (!obj->sym_map)
1416 		return -ENOMEM;
1417 
1418 	for (i = 0; i < n; i++, sym++) {
1419 		/* We already validated all-zero symbol #0 and we already
1420 		 * appended it preventively to the final SYMTAB, so skip it.
1421 		 */
1422 		if (i == 0)
1423 			continue;
1424 
1425 		sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1426 		if (!sym_name) {
1427 			pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1428 			return -EINVAL;
1429 		}
1430 
1431 		err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1432 		if (err)
1433 			return err;
1434 	}
1435 
1436 	return 0;
1437 }
1438 
get_sym_by_idx(struct bpf_linker * linker,size_t sym_idx)1439 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1440 {
1441 	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1442 	Elf64_Sym *syms = symtab->raw_data;
1443 
1444 	return &syms[sym_idx];
1445 }
1446 
find_glob_sym(struct bpf_linker * linker,const char * sym_name)1447 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1448 {
1449 	struct glob_sym *glob_sym;
1450 	const char *name;
1451 	int i;
1452 
1453 	for (i = 0; i < linker->glob_sym_cnt; i++) {
1454 		glob_sym = &linker->glob_syms[i];
1455 		name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1456 
1457 		if (strcmp(name, sym_name) == 0)
1458 			return glob_sym;
1459 	}
1460 
1461 	return NULL;
1462 }
1463 
add_glob_sym(struct bpf_linker * linker)1464 static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1465 {
1466 	struct glob_sym *syms, *sym;
1467 
1468 	syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1469 				   sizeof(*linker->glob_syms));
1470 	if (!syms)
1471 		return NULL;
1472 
1473 	sym = &syms[linker->glob_sym_cnt];
1474 	memset(sym, 0, sizeof(*sym));
1475 	sym->var_idx = -1;
1476 
1477 	linker->glob_syms = syms;
1478 	linker->glob_sym_cnt++;
1479 
1480 	return sym;
1481 }
1482 
glob_sym_btf_matches(const char * sym_name,bool exact,const struct btf * btf1,__u32 id1,const struct btf * btf2,__u32 id2)1483 static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1484 				 const struct btf *btf1, __u32 id1,
1485 				 const struct btf *btf2, __u32 id2)
1486 {
1487 	const struct btf_type *t1, *t2;
1488 	bool is_static1, is_static2;
1489 	const char *n1, *n2;
1490 	int i, n;
1491 
1492 recur:
1493 	n1 = n2 = NULL;
1494 	t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1495 	t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1496 
1497 	/* check if only one side is FWD, otherwise handle with common logic */
1498 	if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1499 		n1 = btf__str_by_offset(btf1, t1->name_off);
1500 		n2 = btf__str_by_offset(btf2, t2->name_off);
1501 		if (strcmp(n1, n2) != 0) {
1502 			pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1503 				sym_name, n1, n2);
1504 			return false;
1505 		}
1506 		/* validate if FWD kind matches concrete kind */
1507 		if (btf_is_fwd(t1)) {
1508 			if (btf_kflag(t1) && btf_is_union(t2))
1509 				return true;
1510 			if (!btf_kflag(t1) && btf_is_struct(t2))
1511 				return true;
1512 			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1513 				sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1514 		} else {
1515 			if (btf_kflag(t2) && btf_is_union(t1))
1516 				return true;
1517 			if (!btf_kflag(t2) && btf_is_struct(t1))
1518 				return true;
1519 			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1520 				sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1521 		}
1522 		return false;
1523 	}
1524 
1525 	if (btf_kind(t1) != btf_kind(t2)) {
1526 		pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1527 			sym_name, btf_kind_str(t1), btf_kind_str(t2));
1528 		return false;
1529 	}
1530 
1531 	switch (btf_kind(t1)) {
1532 	case BTF_KIND_STRUCT:
1533 	case BTF_KIND_UNION:
1534 	case BTF_KIND_ENUM:
1535 	case BTF_KIND_ENUM64:
1536 	case BTF_KIND_FWD:
1537 	case BTF_KIND_FUNC:
1538 	case BTF_KIND_VAR:
1539 		n1 = btf__str_by_offset(btf1, t1->name_off);
1540 		n2 = btf__str_by_offset(btf2, t2->name_off);
1541 		if (strcmp(n1, n2) != 0) {
1542 			pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1543 				sym_name, btf_kind_str(t1), n1, n2);
1544 			return false;
1545 		}
1546 		break;
1547 	default:
1548 		break;
1549 	}
1550 
1551 	switch (btf_kind(t1)) {
1552 	case BTF_KIND_UNKN: /* void */
1553 	case BTF_KIND_FWD:
1554 		return true;
1555 	case BTF_KIND_INT:
1556 	case BTF_KIND_FLOAT:
1557 	case BTF_KIND_ENUM:
1558 	case BTF_KIND_ENUM64:
1559 		/* ignore encoding for int and enum values for enum */
1560 		if (t1->size != t2->size) {
1561 			pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1562 				sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1563 			return false;
1564 		}
1565 		return true;
1566 	case BTF_KIND_PTR:
1567 		/* just validate overall shape of the referenced type, so no
1568 		 * contents comparison for struct/union, and allowed fwd vs
1569 		 * struct/union
1570 		 */
1571 		exact = false;
1572 		id1 = t1->type;
1573 		id2 = t2->type;
1574 		goto recur;
1575 	case BTF_KIND_ARRAY:
1576 		/* ignore index type and array size */
1577 		id1 = btf_array(t1)->type;
1578 		id2 = btf_array(t2)->type;
1579 		goto recur;
1580 	case BTF_KIND_FUNC:
1581 		/* extern and global linkages are compatible */
1582 		is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1583 		is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1584 		if (is_static1 != is_static2) {
1585 			pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1586 			return false;
1587 		}
1588 
1589 		id1 = t1->type;
1590 		id2 = t2->type;
1591 		goto recur;
1592 	case BTF_KIND_VAR:
1593 		/* extern and global linkages are compatible */
1594 		is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1595 		is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1596 		if (is_static1 != is_static2) {
1597 			pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1598 			return false;
1599 		}
1600 
1601 		id1 = t1->type;
1602 		id2 = t2->type;
1603 		goto recur;
1604 	case BTF_KIND_STRUCT:
1605 	case BTF_KIND_UNION: {
1606 		const struct btf_member *m1, *m2;
1607 
1608 		if (!exact)
1609 			return true;
1610 
1611 		if (btf_vlen(t1) != btf_vlen(t2)) {
1612 			pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1613 				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1614 			return false;
1615 		}
1616 
1617 		n = btf_vlen(t1);
1618 		m1 = btf_members(t1);
1619 		m2 = btf_members(t2);
1620 		for (i = 0; i < n; i++, m1++, m2++) {
1621 			n1 = btf__str_by_offset(btf1, m1->name_off);
1622 			n2 = btf__str_by_offset(btf2, m2->name_off);
1623 			if (strcmp(n1, n2) != 0) {
1624 				pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1625 					sym_name, i, n1, n2);
1626 				return false;
1627 			}
1628 			if (m1->offset != m2->offset) {
1629 				pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1630 					sym_name, i, n1);
1631 				return false;
1632 			}
1633 			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1634 				return false;
1635 		}
1636 
1637 		return true;
1638 	}
1639 	case BTF_KIND_FUNC_PROTO: {
1640 		const struct btf_param *m1, *m2;
1641 
1642 		if (btf_vlen(t1) != btf_vlen(t2)) {
1643 			pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1644 				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1645 			return false;
1646 		}
1647 
1648 		n = btf_vlen(t1);
1649 		m1 = btf_params(t1);
1650 		m2 = btf_params(t2);
1651 		for (i = 0; i < n; i++, m1++, m2++) {
1652 			/* ignore func arg names */
1653 			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1654 				return false;
1655 		}
1656 
1657 		/* now check return type as well */
1658 		id1 = t1->type;
1659 		id2 = t2->type;
1660 		goto recur;
1661 	}
1662 
1663 	/* skip_mods_and_typedefs() make this impossible */
1664 	case BTF_KIND_TYPEDEF:
1665 	case BTF_KIND_VOLATILE:
1666 	case BTF_KIND_CONST:
1667 	case BTF_KIND_RESTRICT:
1668 	/* DATASECs are never compared with each other */
1669 	case BTF_KIND_DATASEC:
1670 	default:
1671 		pr_warn("global '%s': unsupported BTF kind %s\n",
1672 			sym_name, btf_kind_str(t1));
1673 		return false;
1674 	}
1675 }
1676 
map_defs_match(const char * sym_name,const struct btf * main_btf,const struct btf_map_def * main_def,const struct btf_map_def * main_inner_def,const struct btf * extra_btf,const struct btf_map_def * extra_def,const struct btf_map_def * extra_inner_def)1677 static bool map_defs_match(const char *sym_name,
1678 			   const struct btf *main_btf,
1679 			   const struct btf_map_def *main_def,
1680 			   const struct btf_map_def *main_inner_def,
1681 			   const struct btf *extra_btf,
1682 			   const struct btf_map_def *extra_def,
1683 			   const struct btf_map_def *extra_inner_def)
1684 {
1685 	const char *reason;
1686 
1687 	if (main_def->map_type != extra_def->map_type) {
1688 		reason = "type";
1689 		goto mismatch;
1690 	}
1691 
1692 	/* check key type/size match */
1693 	if (main_def->key_size != extra_def->key_size) {
1694 		reason = "key_size";
1695 		goto mismatch;
1696 	}
1697 	if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1698 		reason = "key type";
1699 		goto mismatch;
1700 	}
1701 	if ((main_def->parts & MAP_DEF_KEY_TYPE)
1702 	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1703 				      main_btf, main_def->key_type_id,
1704 				      extra_btf, extra_def->key_type_id)) {
1705 		reason = "key type";
1706 		goto mismatch;
1707 	}
1708 
1709 	/* validate value type/size match */
1710 	if (main_def->value_size != extra_def->value_size) {
1711 		reason = "value_size";
1712 		goto mismatch;
1713 	}
1714 	if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1715 		reason = "value type";
1716 		goto mismatch;
1717 	}
1718 	if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1719 	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1720 				      main_btf, main_def->value_type_id,
1721 				      extra_btf, extra_def->value_type_id)) {
1722 		reason = "key type";
1723 		goto mismatch;
1724 	}
1725 
1726 	if (main_def->max_entries != extra_def->max_entries) {
1727 		reason = "max_entries";
1728 		goto mismatch;
1729 	}
1730 	if (main_def->map_flags != extra_def->map_flags) {
1731 		reason = "map_flags";
1732 		goto mismatch;
1733 	}
1734 	if (main_def->numa_node != extra_def->numa_node) {
1735 		reason = "numa_node";
1736 		goto mismatch;
1737 	}
1738 	if (main_def->pinning != extra_def->pinning) {
1739 		reason = "pinning";
1740 		goto mismatch;
1741 	}
1742 
1743 	if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1744 		reason = "inner map";
1745 		goto mismatch;
1746 	}
1747 
1748 	if (main_def->parts & MAP_DEF_INNER_MAP) {
1749 		char inner_map_name[128];
1750 
1751 		snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1752 
1753 		return map_defs_match(inner_map_name,
1754 				      main_btf, main_inner_def, NULL,
1755 				      extra_btf, extra_inner_def, NULL);
1756 	}
1757 
1758 	return true;
1759 
1760 mismatch:
1761 	pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1762 	return false;
1763 }
1764 
glob_map_defs_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,int btf_id)1765 static bool glob_map_defs_match(const char *sym_name,
1766 				struct bpf_linker *linker, struct glob_sym *glob_sym,
1767 				struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1768 {
1769 	struct btf_map_def dst_def = {}, dst_inner_def = {};
1770 	struct btf_map_def src_def = {}, src_inner_def = {};
1771 	const struct btf_type *t;
1772 	int err;
1773 
1774 	t = btf__type_by_id(obj->btf, btf_id);
1775 	if (!btf_is_var(t)) {
1776 		pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1777 		return false;
1778 	}
1779 	t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1780 
1781 	err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1782 	if (err) {
1783 		pr_warn("global '%s': invalid map definition\n", sym_name);
1784 		return false;
1785 	}
1786 
1787 	/* re-parse existing map definition */
1788 	t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1789 	t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1790 	err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1791 	if (err) {
1792 		/* this should not happen, because we already validated it */
1793 		pr_warn("global '%s': invalid dst map definition\n", sym_name);
1794 		return false;
1795 	}
1796 
1797 	/* Currently extern map definition has to be complete and match
1798 	 * concrete map definition exactly. This restriction might be lifted
1799 	 * in the future.
1800 	 */
1801 	return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1802 			      obj->btf, &src_def, &src_inner_def);
1803 }
1804 
glob_syms_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,size_t sym_idx,int btf_id)1805 static bool glob_syms_match(const char *sym_name,
1806 			    struct bpf_linker *linker, struct glob_sym *glob_sym,
1807 			    struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1808 {
1809 	const struct btf_type *src_t;
1810 
1811 	/* if we are dealing with externs, BTF types describing both global
1812 	 * and extern VARs/FUNCs should be completely present in all files
1813 	 */
1814 	if (!glob_sym->btf_id || !btf_id) {
1815 		pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1816 		return false;
1817 	}
1818 
1819 	src_t = btf__type_by_id(obj->btf, btf_id);
1820 	if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1821 		pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1822 			btf_kind_str(src_t), sym_name);
1823 		return false;
1824 	}
1825 
1826 	/* deal with .maps definitions specially */
1827 	if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1828 		return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1829 
1830 	if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1831 				  linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1832 		return false;
1833 
1834 	return true;
1835 }
1836 
btf_is_non_static(const struct btf_type * t)1837 static bool btf_is_non_static(const struct btf_type *t)
1838 {
1839 	return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1840 	       || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1841 }
1842 
find_glob_sym_btf(struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int * out_btf_sec_id,int * out_btf_id)1843 static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1844 			     int *out_btf_sec_id, int *out_btf_id)
1845 {
1846 	int i, j, n, m, btf_id = 0;
1847 	const struct btf_type *t;
1848 	const struct btf_var_secinfo *vi;
1849 	const char *name;
1850 
1851 	if (!obj->btf) {
1852 		pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1853 		return -EINVAL;
1854 	}
1855 
1856 	n = btf__type_cnt(obj->btf);
1857 	for (i = 1; i < n; i++) {
1858 		t = btf__type_by_id(obj->btf, i);
1859 
1860 		/* some global and extern FUNCs and VARs might not be associated with any
1861 		 * DATASEC, so try to detect them in the same pass
1862 		 */
1863 		if (btf_is_non_static(t)) {
1864 			name = btf__str_by_offset(obj->btf, t->name_off);
1865 			if (strcmp(name, sym_name) != 0)
1866 				continue;
1867 
1868 			/* remember and still try to find DATASEC */
1869 			btf_id = i;
1870 			continue;
1871 		}
1872 
1873 		if (!btf_is_datasec(t))
1874 			continue;
1875 
1876 		vi = btf_var_secinfos(t);
1877 		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1878 			t = btf__type_by_id(obj->btf, vi->type);
1879 			name = btf__str_by_offset(obj->btf, t->name_off);
1880 
1881 			if (strcmp(name, sym_name) != 0)
1882 				continue;
1883 			if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1884 				continue;
1885 			if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1886 				continue;
1887 
1888 			if (btf_id && btf_id != vi->type) {
1889 				pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1890 					sym_name, btf_id, vi->type);
1891 				return -EINVAL;
1892 			}
1893 
1894 			*out_btf_sec_id = i;
1895 			*out_btf_id = vi->type;
1896 
1897 			return 0;
1898 		}
1899 	}
1900 
1901 	/* free-floating extern or global FUNC */
1902 	if (btf_id) {
1903 		*out_btf_sec_id = 0;
1904 		*out_btf_id = btf_id;
1905 		return 0;
1906 	}
1907 
1908 	pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1909 	return -ENOENT;
1910 }
1911 
find_src_sec_by_name(struct src_obj * obj,const char * sec_name)1912 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1913 {
1914 	struct src_sec *sec;
1915 	int i;
1916 
1917 	for (i = 1; i < obj->sec_cnt; i++) {
1918 		sec = &obj->secs[i];
1919 
1920 		if (strcmp(sec->sec_name, sec_name) == 0)
1921 			return sec;
1922 	}
1923 
1924 	return NULL;
1925 }
1926 
complete_extern_btf_info(struct btf * dst_btf,int dst_id,struct btf * src_btf,int src_id)1927 static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1928 				    struct btf *src_btf, int src_id)
1929 {
1930 	struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1931 	struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1932 	struct btf_param *src_p, *dst_p;
1933 	const char *s;
1934 	int i, n, off;
1935 
1936 	/* We already made sure that source and destination types (FUNC or
1937 	 * VAR) match in terms of types and argument names.
1938 	 */
1939 	if (btf_is_var(dst_t)) {
1940 		btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1941 		return 0;
1942 	}
1943 
1944 	dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1945 
1946 	/* now onto FUNC_PROTO types */
1947 	src_t = btf_type_by_id(src_btf, src_t->type);
1948 	dst_t = btf_type_by_id(dst_btf, dst_t->type);
1949 
1950 	/* Fill in all the argument names, which for extern FUNCs are missing.
1951 	 * We'll end up with two copies of FUNCs/VARs for externs, but that
1952 	 * will be taken care of by BTF dedup at the very end.
1953 	 * It might be that BTF types for extern in one file has less/more BTF
1954 	 * information (e.g., FWD instead of full STRUCT/UNION information),
1955 	 * but that should be (in most cases, subject to BTF dedup rules)
1956 	 * handled and resolved by BTF dedup algorithm as well, so we won't
1957 	 * worry about it. Our only job is to make sure that argument names
1958 	 * are populated on both sides, otherwise BTF dedup will pedantically
1959 	 * consider them different.
1960 	 */
1961 	src_p = btf_params(src_t);
1962 	dst_p = btf_params(dst_t);
1963 	for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1964 		if (!src_p->name_off)
1965 			continue;
1966 
1967 		/* src_btf has more complete info, so add name to dst_btf */
1968 		s = btf__str_by_offset(src_btf, src_p->name_off);
1969 		off = btf__add_str(dst_btf, s);
1970 		if (off < 0)
1971 			return off;
1972 		dst_p->name_off = off;
1973 	}
1974 	return 0;
1975 }
1976 
sym_update_bind(Elf64_Sym * sym,int sym_bind)1977 static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1978 {
1979 	sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1980 }
1981 
sym_update_type(Elf64_Sym * sym,int sym_type)1982 static void sym_update_type(Elf64_Sym *sym, int sym_type)
1983 {
1984 	sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1985 }
1986 
sym_update_visibility(Elf64_Sym * sym,int sym_vis)1987 static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1988 {
1989 	/* libelf doesn't provide setters for ST_VISIBILITY,
1990 	 * but it is stored in the lower 2 bits of st_other
1991 	 */
1992 	sym->st_other &= ~0x03;
1993 	sym->st_other |= sym_vis;
1994 }
1995 
linker_append_elf_sym(struct bpf_linker * linker,struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int src_sym_idx)1996 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1997 				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1998 {
1999 	struct src_sec *src_sec = NULL;
2000 	struct dst_sec *dst_sec = NULL;
2001 	struct glob_sym *glob_sym = NULL;
2002 	int name_off, sym_type, sym_bind, sym_vis, err;
2003 	int btf_sec_id = 0, btf_id = 0;
2004 	size_t dst_sym_idx;
2005 	Elf64_Sym *dst_sym;
2006 	bool sym_is_extern;
2007 
2008 	sym_type = ELF64_ST_TYPE(sym->st_info);
2009 	sym_bind = ELF64_ST_BIND(sym->st_info);
2010 	sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
2011 	sym_is_extern = sym->st_shndx == SHN_UNDEF;
2012 
2013 	if (sym_is_extern) {
2014 		if (!obj->btf) {
2015 			pr_warn("externs without BTF info are not supported\n");
2016 			return -ENOTSUP;
2017 		}
2018 	} else if (sym->st_shndx < SHN_LORESERVE) {
2019 		src_sec = &obj->secs[sym->st_shndx];
2020 		if (src_sec->skipped)
2021 			return 0;
2022 		dst_sec = &linker->secs[src_sec->dst_id];
2023 
2024 		/* allow only one STT_SECTION symbol per section */
2025 		if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
2026 			obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
2027 			return 0;
2028 		}
2029 	}
2030 
2031 	if (sym_bind == STB_LOCAL)
2032 		goto add_sym;
2033 
2034 	/* find matching BTF info */
2035 	err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
2036 	if (err)
2037 		return err;
2038 
2039 	if (sym_is_extern && btf_sec_id) {
2040 		const char *sec_name = NULL;
2041 		const struct btf_type *t;
2042 
2043 		t = btf__type_by_id(obj->btf, btf_sec_id);
2044 		sec_name = btf__str_by_offset(obj->btf, t->name_off);
2045 
2046 		/* Clang puts unannotated extern vars into
2047 		 * '.extern' BTF DATASEC. Treat them the same
2048 		 * as unannotated extern funcs (which are
2049 		 * currently not put into any DATASECs).
2050 		 * Those don't have associated src_sec/dst_sec.
2051 		 */
2052 		if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
2053 			src_sec = find_src_sec_by_name(obj, sec_name);
2054 			if (!src_sec) {
2055 				pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
2056 				return -ENOENT;
2057 			}
2058 			dst_sec = &linker->secs[src_sec->dst_id];
2059 		}
2060 	}
2061 
2062 	glob_sym = find_glob_sym(linker, sym_name);
2063 	if (glob_sym) {
2064 		/* Preventively resolve to existing symbol. This is
2065 		 * needed for further relocation symbol remapping in
2066 		 * the next step of linking.
2067 		 */
2068 		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2069 
2070 		/* If both symbols are non-externs, at least one of
2071 		 * them has to be STB_WEAK, otherwise they are in
2072 		 * a conflict with each other.
2073 		 */
2074 		if (!sym_is_extern && !glob_sym->is_extern
2075 		    && !glob_sym->is_weak && sym_bind != STB_WEAK) {
2076 			pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
2077 				src_sym_idx, sym_name, obj->filename);
2078 			return -EINVAL;
2079 		}
2080 
2081 		if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
2082 			return -EINVAL;
2083 
2084 		dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
2085 
2086 		/* If new symbol is strong, then force dst_sym to be strong as
2087 		 * well; this way a mix of weak and non-weak extern
2088 		 * definitions will end up being strong.
2089 		 */
2090 		if (sym_bind == STB_GLOBAL) {
2091 			/* We still need to preserve type (NOTYPE or
2092 			 * OBJECT/FUNC, depending on whether the symbol is
2093 			 * extern or not)
2094 			 */
2095 			sym_update_bind(dst_sym, STB_GLOBAL);
2096 			glob_sym->is_weak = false;
2097 		}
2098 
2099 		/* Non-default visibility is "contaminating", with stricter
2100 		 * visibility overwriting more permissive ones, even if more
2101 		 * permissive visibility comes from just an extern definition.
2102 		 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
2103 		 * ensured by ELF symbol sanity checks above.
2104 		 */
2105 		if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
2106 			sym_update_visibility(dst_sym, sym_vis);
2107 
2108 		/* If the new symbol is extern, then regardless if
2109 		 * existing symbol is extern or resolved global, just
2110 		 * keep the existing one untouched.
2111 		 */
2112 		if (sym_is_extern)
2113 			return 0;
2114 
2115 		/* If existing symbol is a strong resolved symbol, bail out,
2116 		 * because we lost resolution battle have nothing to
2117 		 * contribute. We already checked above that there is no
2118 		 * strong-strong conflict. We also already tightened binding
2119 		 * and visibility, so nothing else to contribute at that point.
2120 		 */
2121 		if (!glob_sym->is_extern && sym_bind == STB_WEAK)
2122 			return 0;
2123 
2124 		/* At this point, new symbol is strong non-extern,
2125 		 * so overwrite glob_sym with new symbol information.
2126 		 * Preserve binding and visibility.
2127 		 */
2128 		sym_update_type(dst_sym, sym_type);
2129 		dst_sym->st_shndx = dst_sec->sec_idx;
2130 		dst_sym->st_value = src_sec->dst_off + sym->st_value;
2131 		dst_sym->st_size = sym->st_size;
2132 
2133 		/* see comment below about dst_sec->id vs dst_sec->sec_idx */
2134 		glob_sym->sec_id = dst_sec->id;
2135 		glob_sym->is_extern = false;
2136 
2137 		if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
2138 					     obj->btf, btf_id))
2139 			return -EINVAL;
2140 
2141 		/* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
2142 		glob_sym->underlying_btf_id = 0;
2143 
2144 		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2145 		return 0;
2146 	}
2147 
2148 add_sym:
2149 	name_off = strset__add_str(linker->strtab_strs, sym_name);
2150 	if (name_off < 0)
2151 		return name_off;
2152 
2153 	dst_sym = add_new_sym(linker, &dst_sym_idx);
2154 	if (!dst_sym)
2155 		return -ENOMEM;
2156 
2157 	dst_sym->st_name = name_off;
2158 	dst_sym->st_info = sym->st_info;
2159 	dst_sym->st_other = sym->st_other;
2160 	dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
2161 	dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
2162 	dst_sym->st_size = sym->st_size;
2163 
2164 	obj->sym_map[src_sym_idx] = dst_sym_idx;
2165 
2166 	if (sym_type == STT_SECTION && dst_sym) {
2167 		dst_sec->sec_sym_idx = dst_sym_idx;
2168 		dst_sym->st_value = 0;
2169 	}
2170 
2171 	if (sym_bind != STB_LOCAL) {
2172 		glob_sym = add_glob_sym(linker);
2173 		if (!glob_sym)
2174 			return -ENOMEM;
2175 
2176 		glob_sym->sym_idx = dst_sym_idx;
2177 		/* we use dst_sec->id (and not dst_sec->sec_idx), because
2178 		 * ephemeral sections (.kconfig, .ksyms, etc) don't have
2179 		 * sec_idx (as they don't have corresponding ELF section), but
2180 		 * still have id. .extern doesn't have even ephemeral section
2181 		 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
2182 		 */
2183 		glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
2184 		glob_sym->name_off = name_off;
2185 		/* we will fill btf_id in during BTF merging step */
2186 		glob_sym->btf_id = 0;
2187 		glob_sym->is_extern = sym_is_extern;
2188 		glob_sym->is_weak = sym_bind == STB_WEAK;
2189 	}
2190 
2191 	return 0;
2192 }
2193 
linker_append_elf_relos(struct bpf_linker * linker,struct src_obj * obj)2194 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2195 {
2196 	struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2197 	int i, err;
2198 
2199 	for (i = 1; i < obj->sec_cnt; i++) {
2200 		struct src_sec *src_sec, *src_linked_sec;
2201 		struct dst_sec *dst_sec, *dst_linked_sec;
2202 		Elf64_Rel *src_rel, *dst_rel;
2203 		int j, n;
2204 
2205 		src_sec = &obj->secs[i];
2206 		if (!is_relo_sec(src_sec))
2207 			continue;
2208 
2209 		/* shdr->sh_info points to relocatable section */
2210 		src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2211 		if (src_linked_sec->skipped)
2212 			continue;
2213 
2214 		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2215 		if (!dst_sec) {
2216 			dst_sec = add_dst_sec(linker, src_sec->sec_name);
2217 			if (!dst_sec)
2218 				return -ENOMEM;
2219 			err = init_sec(linker, dst_sec, src_sec);
2220 			if (err) {
2221 				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2222 				return err;
2223 			}
2224 		} else if (!secs_match(dst_sec, src_sec)) {
2225 			pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2226 			return -1;
2227 		}
2228 
2229 		/* shdr->sh_link points to SYMTAB */
2230 		dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2231 
2232 		/* shdr->sh_info points to relocated section */
2233 		dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2234 		dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2235 
2236 		src_sec->dst_id = dst_sec->id;
2237 		err = extend_sec(linker, dst_sec, src_sec);
2238 		if (err)
2239 			return err;
2240 
2241 		src_rel = src_sec->data->d_buf;
2242 		dst_rel = dst_sec->raw_data + src_sec->dst_off;
2243 		n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2244 		for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2245 			size_t src_sym_idx, dst_sym_idx, sym_type;
2246 			Elf64_Sym *src_sym;
2247 
2248 			src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2249 			src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2250 
2251 			dst_sym_idx = obj->sym_map[src_sym_idx];
2252 			dst_rel->r_offset += src_linked_sec->dst_off;
2253 			sym_type = ELF64_R_TYPE(src_rel->r_info);
2254 			dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2255 
2256 			if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2257 				struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2258 				struct bpf_insn *insn;
2259 
2260 				if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2261 					/* calls to the very first static function inside
2262 					 * .text section at offset 0 will
2263 					 * reference section symbol, not the
2264 					 * function symbol. Fix that up,
2265 					 * otherwise it won't be possible to
2266 					 * relocate calls to two different
2267 					 * static functions with the same name
2268 					 * (rom two different object files)
2269 					 */
2270 					insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2271 					if (insn->code == (BPF_JMP | BPF_CALL))
2272 						insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2273 					else
2274 						insn->imm += sec->dst_off;
2275 				} else {
2276 					pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2277 					return -EINVAL;
2278 				}
2279 			}
2280 
2281 		}
2282 	}
2283 
2284 	return 0;
2285 }
2286 
find_sym_by_name(struct src_obj * obj,size_t sec_idx,int sym_type,const char * sym_name)2287 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2288 				   int sym_type, const char *sym_name)
2289 {
2290 	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2291 	Elf64_Sym *sym = symtab->data->d_buf;
2292 	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2293 	int str_sec_idx = symtab->shdr->sh_link;
2294 	const char *name;
2295 
2296 	for (i = 0; i < n; i++, sym++) {
2297 		if (sym->st_shndx != sec_idx)
2298 			continue;
2299 		if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2300 			continue;
2301 
2302 		name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2303 		if (!name)
2304 			return NULL;
2305 
2306 		if (strcmp(sym_name, name) != 0)
2307 			continue;
2308 
2309 		return sym;
2310 	}
2311 
2312 	return NULL;
2313 }
2314 
linker_fixup_btf(struct src_obj * obj)2315 static int linker_fixup_btf(struct src_obj *obj)
2316 {
2317 	const char *sec_name;
2318 	struct src_sec *sec;
2319 	int i, j, n, m;
2320 
2321 	if (!obj->btf)
2322 		return 0;
2323 
2324 	n = btf__type_cnt(obj->btf);
2325 	for (i = 1; i < n; i++) {
2326 		struct btf_var_secinfo *vi;
2327 		struct btf_type *t;
2328 
2329 		t = btf_type_by_id(obj->btf, i);
2330 		if (btf_kind(t) != BTF_KIND_DATASEC)
2331 			continue;
2332 
2333 		sec_name = btf__str_by_offset(obj->btf, t->name_off);
2334 		sec = find_src_sec_by_name(obj, sec_name);
2335 		if (sec) {
2336 			/* record actual section size, unless ephemeral */
2337 			if (sec->shdr)
2338 				t->size = sec->shdr->sh_size;
2339 		} else {
2340 			/* BTF can have some sections that are not represented
2341 			 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2342 			 * for special extern variables.
2343 			 *
2344 			 * For all but one such special (ephemeral)
2345 			 * sections, we pre-create "section shells" to be able
2346 			 * to keep track of extra per-section metadata later
2347 			 * (e.g., those BTF extern variables).
2348 			 *
2349 			 * .extern is even more special, though, because it
2350 			 * contains extern variables that need to be resolved
2351 			 * by static linker, not libbpf and kernel. When such
2352 			 * externs are resolved, we are going to remove them
2353 			 * from .extern BTF section and might end up not
2354 			 * needing it at all. Each resolved extern should have
2355 			 * matching non-extern VAR/FUNC in other sections.
2356 			 *
2357 			 * We do support leaving some of the externs
2358 			 * unresolved, though, to support cases of building
2359 			 * libraries, which will later be linked against final
2360 			 * BPF applications. So if at finalization we still
2361 			 * see unresolved externs, we'll create .extern
2362 			 * section on our own.
2363 			 */
2364 			if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2365 				continue;
2366 
2367 			sec = add_src_sec(obj, sec_name);
2368 			if (!sec)
2369 				return -ENOMEM;
2370 
2371 			sec->ephemeral = true;
2372 			sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2373 		}
2374 
2375 		/* remember ELF section and its BTF type ID match */
2376 		sec->sec_type_id = i;
2377 
2378 		/* fix up variable offsets */
2379 		vi = btf_var_secinfos(t);
2380 		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2381 			const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2382 			const char *var_name;
2383 			int var_linkage;
2384 			Elf64_Sym *sym;
2385 
2386 			/* could be a variable or function */
2387 			if (!btf_is_var(vt))
2388 				continue;
2389 
2390 			var_name = btf__str_by_offset(obj->btf, vt->name_off);
2391 			var_linkage = btf_var(vt)->linkage;
2392 
2393 			/* no need to patch up static or extern vars */
2394 			if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2395 				continue;
2396 
2397 			sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2398 			if (!sym) {
2399 				pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2400 				return -ENOENT;
2401 			}
2402 
2403 			vi->offset = sym->st_value;
2404 		}
2405 	}
2406 
2407 	return 0;
2408 }
2409 
linker_append_btf(struct bpf_linker * linker,struct src_obj * obj)2410 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2411 {
2412 	const struct btf_type *t;
2413 	int i, j, n, start_id, id, err;
2414 	const char *name;
2415 
2416 	if (!obj->btf)
2417 		return 0;
2418 
2419 	start_id = btf__type_cnt(linker->btf);
2420 	n = btf__type_cnt(obj->btf);
2421 
2422 	obj->btf_type_map = calloc(n + 1, sizeof(int));
2423 	if (!obj->btf_type_map)
2424 		return -ENOMEM;
2425 
2426 	for (i = 1; i < n; i++) {
2427 		struct glob_sym *glob_sym = NULL;
2428 
2429 		t = btf__type_by_id(obj->btf, i);
2430 
2431 		/* DATASECs are handled specially below */
2432 		if (btf_kind(t) == BTF_KIND_DATASEC)
2433 			continue;
2434 
2435 		if (btf_is_non_static(t)) {
2436 			/* there should be glob_sym already */
2437 			name = btf__str_by_offset(obj->btf, t->name_off);
2438 			glob_sym = find_glob_sym(linker, name);
2439 
2440 			/* VARs without corresponding glob_sym are those that
2441 			 * belong to skipped/deduplicated sections (i.e.,
2442 			 * license and version), so just skip them
2443 			 */
2444 			if (!glob_sym)
2445 				continue;
2446 
2447 			/* linker_append_elf_sym() might have requested
2448 			 * updating underlying type ID, if extern was resolved
2449 			 * to strong symbol or weak got upgraded to non-weak
2450 			 */
2451 			if (glob_sym->underlying_btf_id == 0)
2452 				glob_sym->underlying_btf_id = -t->type;
2453 
2454 			/* globals from previous object files that match our
2455 			 * VAR/FUNC already have a corresponding associated
2456 			 * BTF type, so just make sure to use it
2457 			 */
2458 			if (glob_sym->btf_id) {
2459 				/* reuse existing BTF type for global var/func */
2460 				obj->btf_type_map[i] = glob_sym->btf_id;
2461 				continue;
2462 			}
2463 		}
2464 
2465 		id = btf__add_type(linker->btf, obj->btf, t);
2466 		if (id < 0) {
2467 			pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2468 			return id;
2469 		}
2470 
2471 		obj->btf_type_map[i] = id;
2472 
2473 		/* record just appended BTF type for var/func */
2474 		if (glob_sym) {
2475 			glob_sym->btf_id = id;
2476 			glob_sym->underlying_btf_id = -t->type;
2477 		}
2478 	}
2479 
2480 	/* remap all the types except DATASECs */
2481 	n = btf__type_cnt(linker->btf);
2482 	for (i = start_id; i < n; i++) {
2483 		struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2484 		struct btf_field_iter it;
2485 		__u32 *type_id;
2486 
2487 		err = btf_field_iter_init(&it, dst_t, BTF_FIELD_ITER_IDS);
2488 		if (err)
2489 			return err;
2490 
2491 		while ((type_id = btf_field_iter_next(&it))) {
2492 			int new_id = obj->btf_type_map[*type_id];
2493 
2494 			/* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2495 			if (new_id == 0 && *type_id != 0) {
2496 				pr_warn("failed to find new ID mapping for original BTF type ID %u\n",
2497 					*type_id);
2498 				return -EINVAL;
2499 			}
2500 
2501 			*type_id = obj->btf_type_map[*type_id];
2502 		}
2503 	}
2504 
2505 	/* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2506 	 * actual type), if necessary
2507 	 */
2508 	for (i = 0; i < linker->glob_sym_cnt; i++) {
2509 		struct glob_sym *glob_sym = &linker->glob_syms[i];
2510 		struct btf_type *glob_t;
2511 
2512 		if (glob_sym->underlying_btf_id >= 0)
2513 			continue;
2514 
2515 		glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2516 
2517 		glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2518 		glob_t->type = glob_sym->underlying_btf_id;
2519 	}
2520 
2521 	/* append DATASEC info */
2522 	for (i = 1; i < obj->sec_cnt; i++) {
2523 		struct src_sec *src_sec;
2524 		struct dst_sec *dst_sec;
2525 		const struct btf_var_secinfo *src_var;
2526 		struct btf_var_secinfo *dst_var;
2527 
2528 		src_sec = &obj->secs[i];
2529 		if (!src_sec->sec_type_id || src_sec->skipped)
2530 			continue;
2531 		dst_sec = &linker->secs[src_sec->dst_id];
2532 
2533 		/* Mark section as having BTF regardless of the presence of
2534 		 * variables. In some cases compiler might generate empty BTF
2535 		 * with no variables information. E.g., when promoting local
2536 		 * array/structure variable initial values and BPF object
2537 		 * file otherwise has no read-only static variables in
2538 		 * .rodata. We need to preserve such empty BTF and just set
2539 		 * correct section size.
2540 		 */
2541 		dst_sec->has_btf = true;
2542 
2543 		t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2544 		src_var = btf_var_secinfos(t);
2545 		n = btf_vlen(t);
2546 		for (j = 0; j < n; j++, src_var++) {
2547 			void *sec_vars = dst_sec->sec_vars;
2548 			int new_id = obj->btf_type_map[src_var->type];
2549 			struct glob_sym *glob_sym = NULL;
2550 
2551 			t = btf_type_by_id(linker->btf, new_id);
2552 			if (btf_is_non_static(t)) {
2553 				name = btf__str_by_offset(linker->btf, t->name_off);
2554 				glob_sym = find_glob_sym(linker, name);
2555 				if (glob_sym->sec_id != dst_sec->id) {
2556 					pr_warn("global '%s': section mismatch %d vs %d\n",
2557 						name, glob_sym->sec_id, dst_sec->id);
2558 					return -EINVAL;
2559 				}
2560 			}
2561 
2562 			/* If there is already a member (VAR or FUNC) mapped
2563 			 * to the same type, don't add a duplicate entry.
2564 			 * This will happen when multiple object files define
2565 			 * the same extern VARs/FUNCs.
2566 			 */
2567 			if (glob_sym && glob_sym->var_idx >= 0) {
2568 				__s64 sz;
2569 
2570 				/* FUNCs don't have size, nothing to update */
2571 				if (btf_is_func(t))
2572 					continue;
2573 
2574 				dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2575 				/* Because underlying BTF type might have
2576 				 * changed, so might its size have changed, so
2577 				 * re-calculate and update it in sec_var.
2578 				 */
2579 				sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2580 				if (sz < 0) {
2581 					pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2582 						name, (int)sz);
2583 					return -EINVAL;
2584 				}
2585 				dst_var->size = sz;
2586 				continue;
2587 			}
2588 
2589 			sec_vars = libbpf_reallocarray(sec_vars,
2590 						       dst_sec->sec_var_cnt + 1,
2591 						       sizeof(*dst_sec->sec_vars));
2592 			if (!sec_vars)
2593 				return -ENOMEM;
2594 
2595 			dst_sec->sec_vars = sec_vars;
2596 			dst_sec->sec_var_cnt++;
2597 
2598 			dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2599 			dst_var->type = obj->btf_type_map[src_var->type];
2600 			dst_var->size = src_var->size;
2601 			dst_var->offset = src_sec->dst_off + src_var->offset;
2602 
2603 			if (glob_sym)
2604 				glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2605 		}
2606 	}
2607 
2608 	return 0;
2609 }
2610 
add_btf_ext_rec(struct btf_ext_sec_data * ext_data,const void * src_rec)2611 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2612 {
2613 	void *tmp;
2614 
2615 	tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2616 	if (!tmp)
2617 		return NULL;
2618 	ext_data->recs = tmp;
2619 
2620 	tmp += ext_data->rec_cnt * ext_data->rec_sz;
2621 	memcpy(tmp, src_rec, ext_data->rec_sz);
2622 
2623 	ext_data->rec_cnt++;
2624 
2625 	return tmp;
2626 }
2627 
linker_append_btf_ext(struct bpf_linker * linker,struct src_obj * obj)2628 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2629 {
2630 	const struct btf_ext_info_sec *ext_sec;
2631 	const char *sec_name, *s;
2632 	struct src_sec *src_sec;
2633 	struct dst_sec *dst_sec;
2634 	int rec_sz, str_off, i;
2635 
2636 	if (!obj->btf_ext)
2637 		return 0;
2638 
2639 	rec_sz = obj->btf_ext->func_info.rec_size;
2640 	for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2641 		struct bpf_func_info_min *src_rec, *dst_rec;
2642 
2643 		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2644 		src_sec = find_src_sec_by_name(obj, sec_name);
2645 		if (!src_sec) {
2646 			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2647 			return -EINVAL;
2648 		}
2649 		dst_sec = &linker->secs[src_sec->dst_id];
2650 
2651 		if (dst_sec->func_info.rec_sz == 0)
2652 			dst_sec->func_info.rec_sz = rec_sz;
2653 		if (dst_sec->func_info.rec_sz != rec_sz) {
2654 			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2655 			return -EINVAL;
2656 		}
2657 
2658 		for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2659 			dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2660 			if (!dst_rec)
2661 				return -ENOMEM;
2662 
2663 			dst_rec->insn_off += src_sec->dst_off;
2664 			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2665 		}
2666 	}
2667 
2668 	rec_sz = obj->btf_ext->line_info.rec_size;
2669 	for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2670 		struct bpf_line_info_min *src_rec, *dst_rec;
2671 
2672 		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2673 		src_sec = find_src_sec_by_name(obj, sec_name);
2674 		if (!src_sec) {
2675 			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2676 			return -EINVAL;
2677 		}
2678 		dst_sec = &linker->secs[src_sec->dst_id];
2679 
2680 		if (dst_sec->line_info.rec_sz == 0)
2681 			dst_sec->line_info.rec_sz = rec_sz;
2682 		if (dst_sec->line_info.rec_sz != rec_sz) {
2683 			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2684 			return -EINVAL;
2685 		}
2686 
2687 		for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2688 			dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2689 			if (!dst_rec)
2690 				return -ENOMEM;
2691 
2692 			dst_rec->insn_off += src_sec->dst_off;
2693 
2694 			s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2695 			str_off = btf__add_str(linker->btf, s);
2696 			if (str_off < 0)
2697 				return -ENOMEM;
2698 			dst_rec->file_name_off = str_off;
2699 
2700 			s = btf__str_by_offset(obj->btf, src_rec->line_off);
2701 			str_off = btf__add_str(linker->btf, s);
2702 			if (str_off < 0)
2703 				return -ENOMEM;
2704 			dst_rec->line_off = str_off;
2705 
2706 			/* dst_rec->line_col is fine */
2707 		}
2708 	}
2709 
2710 	rec_sz = obj->btf_ext->core_relo_info.rec_size;
2711 	for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2712 		struct bpf_core_relo *src_rec, *dst_rec;
2713 
2714 		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2715 		src_sec = find_src_sec_by_name(obj, sec_name);
2716 		if (!src_sec) {
2717 			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2718 			return -EINVAL;
2719 		}
2720 		dst_sec = &linker->secs[src_sec->dst_id];
2721 
2722 		if (dst_sec->core_relo_info.rec_sz == 0)
2723 			dst_sec->core_relo_info.rec_sz = rec_sz;
2724 		if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2725 			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2726 			return -EINVAL;
2727 		}
2728 
2729 		for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2730 			dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2731 			if (!dst_rec)
2732 				return -ENOMEM;
2733 
2734 			dst_rec->insn_off += src_sec->dst_off;
2735 			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2736 
2737 			s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2738 			str_off = btf__add_str(linker->btf, s);
2739 			if (str_off < 0)
2740 				return -ENOMEM;
2741 			dst_rec->access_str_off = str_off;
2742 
2743 			/* dst_rec->kind is fine */
2744 		}
2745 	}
2746 
2747 	return 0;
2748 }
2749 
bpf_linker__finalize(struct bpf_linker * linker)2750 int bpf_linker__finalize(struct bpf_linker *linker)
2751 {
2752 	struct dst_sec *sec;
2753 	size_t strs_sz;
2754 	const void *strs;
2755 	int err, i;
2756 
2757 	if (!linker->elf)
2758 		return libbpf_err(-EINVAL);
2759 
2760 	err = finalize_btf(linker);
2761 	if (err)
2762 		return libbpf_err(err);
2763 
2764 	/* Finalize strings */
2765 	strs_sz = strset__data_size(linker->strtab_strs);
2766 	strs = strset__data(linker->strtab_strs);
2767 
2768 	sec = &linker->secs[linker->strtab_sec_idx];
2769 	sec->data->d_align = 1;
2770 	sec->data->d_off = 0LL;
2771 	sec->data->d_buf = (void *)strs;
2772 	sec->data->d_type = ELF_T_BYTE;
2773 	sec->data->d_size = strs_sz;
2774 	sec->shdr->sh_size = strs_sz;
2775 
2776 	for (i = 1; i < linker->sec_cnt; i++) {
2777 		sec = &linker->secs[i];
2778 
2779 		/* STRTAB is handled specially above */
2780 		if (sec->sec_idx == linker->strtab_sec_idx)
2781 			continue;
2782 
2783 		/* special ephemeral sections (.ksyms, .kconfig, etc) */
2784 		if (!sec->scn)
2785 			continue;
2786 
2787 		/* restore sections with bpf insns to target byte-order */
2788 		if (linker->swapped_endian && is_exec_sec(sec))
2789 			exec_sec_bswap(sec->raw_data, sec->sec_sz);
2790 
2791 		sec->data->d_buf = sec->raw_data;
2792 	}
2793 
2794 	/* Finalize ELF layout */
2795 	if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2796 		err = -EINVAL;
2797 		pr_warn_elf("failed to finalize ELF layout");
2798 		return libbpf_err(err);
2799 	}
2800 
2801 	/* Write out final ELF contents */
2802 	if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2803 		err = -EINVAL;
2804 		pr_warn_elf("failed to write ELF contents");
2805 		return libbpf_err(err);
2806 	}
2807 
2808 	elf_end(linker->elf);
2809 	linker->elf = NULL;
2810 
2811 	if (linker->fd_is_owned)
2812 		close(linker->fd);
2813 	linker->fd = -1;
2814 
2815 	return 0;
2816 }
2817 
emit_elf_data_sec(struct bpf_linker * linker,const char * sec_name,size_t align,const void * raw_data,size_t raw_sz)2818 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2819 			     size_t align, const void *raw_data, size_t raw_sz)
2820 {
2821 	Elf_Scn *scn;
2822 	Elf_Data *data;
2823 	Elf64_Shdr *shdr;
2824 	int name_off;
2825 
2826 	name_off = strset__add_str(linker->strtab_strs, sec_name);
2827 	if (name_off < 0)
2828 		return name_off;
2829 
2830 	scn = elf_newscn(linker->elf);
2831 	if (!scn)
2832 		return -ENOMEM;
2833 	data = elf_newdata(scn);
2834 	if (!data)
2835 		return -ENOMEM;
2836 	shdr = elf64_getshdr(scn);
2837 	if (!shdr)
2838 		return -EINVAL;
2839 
2840 	shdr->sh_name = name_off;
2841 	shdr->sh_type = SHT_PROGBITS;
2842 	shdr->sh_flags = 0;
2843 	shdr->sh_size = raw_sz;
2844 	shdr->sh_link = 0;
2845 	shdr->sh_info = 0;
2846 	shdr->sh_addralign = align;
2847 	shdr->sh_entsize = 0;
2848 
2849 	data->d_type = ELF_T_BYTE;
2850 	data->d_size = raw_sz;
2851 	data->d_buf = (void *)raw_data;
2852 	data->d_align = align;
2853 	data->d_off = 0;
2854 
2855 	return 0;
2856 }
2857 
finalize_btf(struct bpf_linker * linker)2858 static int finalize_btf(struct bpf_linker *linker)
2859 {
2860 	enum btf_endianness link_endianness;
2861 	LIBBPF_OPTS(btf_dedup_opts, opts);
2862 	struct btf *btf = linker->btf;
2863 	const void *raw_data;
2864 	int i, j, id, err;
2865 	__u32 raw_sz;
2866 
2867 	/* bail out if no BTF data was produced */
2868 	if (btf__type_cnt(linker->btf) == 1)
2869 		return 0;
2870 
2871 	for (i = 1; i < linker->sec_cnt; i++) {
2872 		struct dst_sec *sec = &linker->secs[i];
2873 
2874 		if (!sec->has_btf)
2875 			continue;
2876 
2877 		id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2878 		if (id < 0) {
2879 			pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2880 				sec->sec_name, id);
2881 			return id;
2882 		}
2883 
2884 		for (j = 0; j < sec->sec_var_cnt; j++) {
2885 			struct btf_var_secinfo *vi = &sec->sec_vars[j];
2886 
2887 			if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2888 				return -EINVAL;
2889 		}
2890 	}
2891 
2892 	err = finalize_btf_ext(linker);
2893 	if (err) {
2894 		pr_warn(".BTF.ext generation failed: %s\n", errstr(err));
2895 		return err;
2896 	}
2897 
2898 	opts.btf_ext = linker->btf_ext;
2899 	err = btf__dedup(linker->btf, &opts);
2900 	if (err) {
2901 		pr_warn("BTF dedup failed: %s\n", errstr(err));
2902 		return err;
2903 	}
2904 
2905 	/* Set .BTF and .BTF.ext output byte order */
2906 	link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ?
2907 			  BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
2908 	btf__set_endianness(linker->btf, link_endianness);
2909 	if (linker->btf_ext)
2910 		btf_ext__set_endianness(linker->btf_ext, link_endianness);
2911 
2912 	/* Emit .BTF section */
2913 	raw_data = btf__raw_data(linker->btf, &raw_sz);
2914 	if (!raw_data)
2915 		return -ENOMEM;
2916 
2917 	err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2918 	if (err) {
2919 		pr_warn("failed to write out .BTF ELF section: %s\n", errstr(err));
2920 		return err;
2921 	}
2922 
2923 	/* Emit .BTF.ext section */
2924 	if (linker->btf_ext) {
2925 		raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);
2926 		if (!raw_data)
2927 			return -ENOMEM;
2928 
2929 		err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2930 		if (err) {
2931 			pr_warn("failed to write out .BTF.ext ELF section: %s\n", errstr(err));
2932 			return err;
2933 		}
2934 	}
2935 
2936 	return 0;
2937 }
2938 
emit_btf_ext_data(struct bpf_linker * linker,void * output,const char * sec_name,struct btf_ext_sec_data * sec_data)2939 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2940 			     const char *sec_name, struct btf_ext_sec_data *sec_data)
2941 {
2942 	struct btf_ext_info_sec *sec_info;
2943 	void *cur = output;
2944 	int str_off;
2945 	size_t sz;
2946 
2947 	if (!sec_data->rec_cnt)
2948 		return 0;
2949 
2950 	str_off = btf__add_str(linker->btf, sec_name);
2951 	if (str_off < 0)
2952 		return -ENOMEM;
2953 
2954 	sec_info = cur;
2955 	sec_info->sec_name_off = str_off;
2956 	sec_info->num_info = sec_data->rec_cnt;
2957 	cur += sizeof(struct btf_ext_info_sec);
2958 
2959 	sz = sec_data->rec_cnt * sec_data->rec_sz;
2960 	memcpy(cur, sec_data->recs, sz);
2961 	cur += sz;
2962 
2963 	return cur - output;
2964 }
2965 
finalize_btf_ext(struct bpf_linker * linker)2966 static int finalize_btf_ext(struct bpf_linker *linker)
2967 {
2968 	size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2969 	size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2970 	struct btf_ext_header *hdr;
2971 	void *data, *cur;
2972 	int i, err, sz;
2973 
2974 	/* validate that all sections have the same .BTF.ext record sizes
2975 	 * and calculate total data size for each type of data (func info,
2976 	 * line info, core relos)
2977 	 */
2978 	for (i = 1; i < linker->sec_cnt; i++) {
2979 		struct dst_sec *sec = &linker->secs[i];
2980 
2981 		if (sec->func_info.rec_cnt) {
2982 			if (func_rec_sz == 0)
2983 				func_rec_sz = sec->func_info.rec_sz;
2984 			if (func_rec_sz != sec->func_info.rec_sz) {
2985 				pr_warn("mismatch in func_info record size %zu != %u\n",
2986 					func_rec_sz, sec->func_info.rec_sz);
2987 				return -EINVAL;
2988 			}
2989 
2990 			funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2991 		}
2992 		if (sec->line_info.rec_cnt) {
2993 			if (line_rec_sz == 0)
2994 				line_rec_sz = sec->line_info.rec_sz;
2995 			if (line_rec_sz != sec->line_info.rec_sz) {
2996 				pr_warn("mismatch in line_info record size %zu != %u\n",
2997 					line_rec_sz, sec->line_info.rec_sz);
2998 				return -EINVAL;
2999 			}
3000 
3001 			lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
3002 		}
3003 		if (sec->core_relo_info.rec_cnt) {
3004 			if (core_relo_rec_sz == 0)
3005 				core_relo_rec_sz = sec->core_relo_info.rec_sz;
3006 			if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
3007 				pr_warn("mismatch in core_relo_info record size %zu != %u\n",
3008 					core_relo_rec_sz, sec->core_relo_info.rec_sz);
3009 				return -EINVAL;
3010 			}
3011 
3012 			core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
3013 		}
3014 	}
3015 
3016 	if (!funcs_sz && !lines_sz && !core_relos_sz)
3017 		return 0;
3018 
3019 	total_sz += sizeof(struct btf_ext_header);
3020 	if (funcs_sz) {
3021 		funcs_sz += sizeof(__u32); /* record size prefix */
3022 		total_sz += funcs_sz;
3023 	}
3024 	if (lines_sz) {
3025 		lines_sz += sizeof(__u32); /* record size prefix */
3026 		total_sz += lines_sz;
3027 	}
3028 	if (core_relos_sz) {
3029 		core_relos_sz += sizeof(__u32); /* record size prefix */
3030 		total_sz += core_relos_sz;
3031 	}
3032 
3033 	cur = data = calloc(1, total_sz);
3034 	if (!data)
3035 		return -ENOMEM;
3036 
3037 	hdr = cur;
3038 	hdr->magic = BTF_MAGIC;
3039 	hdr->version = BTF_VERSION;
3040 	hdr->flags = 0;
3041 	hdr->hdr_len = sizeof(struct btf_ext_header);
3042 	cur += sizeof(struct btf_ext_header);
3043 
3044 	/* All offsets are in bytes relative to the end of this header */
3045 	hdr->func_info_off = 0;
3046 	hdr->func_info_len = funcs_sz;
3047 	hdr->line_info_off = funcs_sz;
3048 	hdr->line_info_len = lines_sz;
3049 	hdr->core_relo_off = funcs_sz + lines_sz;
3050 	hdr->core_relo_len = core_relos_sz;
3051 
3052 	if (funcs_sz) {
3053 		*(__u32 *)cur = func_rec_sz;
3054 		cur += sizeof(__u32);
3055 
3056 		for (i = 1; i < linker->sec_cnt; i++) {
3057 			struct dst_sec *sec = &linker->secs[i];
3058 
3059 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
3060 			if (sz < 0) {
3061 				err = sz;
3062 				goto out;
3063 			}
3064 
3065 			cur += sz;
3066 		}
3067 	}
3068 
3069 	if (lines_sz) {
3070 		*(__u32 *)cur = line_rec_sz;
3071 		cur += sizeof(__u32);
3072 
3073 		for (i = 1; i < linker->sec_cnt; i++) {
3074 			struct dst_sec *sec = &linker->secs[i];
3075 
3076 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
3077 			if (sz < 0) {
3078 				err = sz;
3079 				goto out;
3080 			}
3081 
3082 			cur += sz;
3083 		}
3084 	}
3085 
3086 	if (core_relos_sz) {
3087 		*(__u32 *)cur = core_relo_rec_sz;
3088 		cur += sizeof(__u32);
3089 
3090 		for (i = 1; i < linker->sec_cnt; i++) {
3091 			struct dst_sec *sec = &linker->secs[i];
3092 
3093 			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
3094 			if (sz < 0) {
3095 				err = sz;
3096 				goto out;
3097 			}
3098 
3099 			cur += sz;
3100 		}
3101 	}
3102 
3103 	linker->btf_ext = btf_ext__new(data, total_sz);
3104 	err = libbpf_get_error(linker->btf_ext);
3105 	if (err) {
3106 		linker->btf_ext = NULL;
3107 		pr_warn("failed to parse final .BTF.ext data: %s\n", errstr(err));
3108 		goto out;
3109 	}
3110 
3111 out:
3112 	free(data);
3113 	return err;
3114 }
3115