1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2 /*
3 * BPF static linker
4 *
5 * Copyright (c) 2021 Facebook
6 */
7 #ifndef _GNU_SOURCE
8 #define _GNU_SOURCE
9 #endif
10
11 #include <stdbool.h>
12 #include <stddef.h>
13 #include <stdio.h>
14 #include <stdlib.h>
15 #include <string.h>
16 #include <unistd.h>
17 #include <errno.h>
18 #include <linux/err.h>
19 #include <linux/btf.h>
20 #include <elf.h>
21 #include <libelf.h>
22 #include <fcntl.h>
23 #include <sys/mman.h>
24 #include "libbpf.h"
25 #include "btf.h"
26 #include "libbpf_internal.h"
27 #include "strset.h"
28 #include "str_error.h"
29
30 #define BTF_EXTERN_SEC ".extern"
31
32 struct src_sec {
33 const char *sec_name;
34 /* positional (not necessarily ELF) index in an array of sections */
35 int id;
36 /* positional (not necessarily ELF) index of a matching section in a final object file */
37 int dst_id;
38 /* section data offset in a matching output section */
39 int dst_off;
40 /* whether section is omitted from the final ELF file */
41 bool skipped;
42 /* whether section is an ephemeral section, not mapped to an ELF section */
43 bool ephemeral;
44
45 /* ELF info */
46 size_t sec_idx;
47 Elf_Scn *scn;
48 Elf64_Shdr *shdr;
49 Elf_Data *data;
50
51 /* corresponding BTF DATASEC type ID */
52 int sec_type_id;
53 };
54
55 struct src_obj {
56 const char *filename;
57 int fd;
58 Elf *elf;
59 /* Section header strings section index */
60 size_t shstrs_sec_idx;
61 /* SYMTAB section index */
62 size_t symtab_sec_idx;
63
64 struct btf *btf;
65 struct btf_ext *btf_ext;
66
67 /* List of sections (including ephemeral). Slot zero is unused. */
68 struct src_sec *secs;
69 int sec_cnt;
70
71 /* mapping of symbol indices from src to dst ELF */
72 int *sym_map;
73 /* mapping from the src BTF type IDs to dst ones */
74 int *btf_type_map;
75 };
76
77 /* single .BTF.ext data section */
78 struct btf_ext_sec_data {
79 size_t rec_cnt;
80 __u32 rec_sz;
81 void *recs;
82 };
83
84 struct glob_sym {
85 /* ELF symbol index */
86 int sym_idx;
87 /* associated section id for .ksyms, .kconfig, etc, but not .extern */
88 int sec_id;
89 /* extern name offset in STRTAB */
90 int name_off;
91 /* optional associated BTF type ID */
92 int btf_id;
93 /* BTF type ID to which VAR/FUNC type is pointing to; used for
94 * rewriting types when extern VAR/FUNC is resolved to a concrete
95 * definition
96 */
97 int underlying_btf_id;
98 /* sec_var index in the corresponding dst_sec, if exists */
99 int var_idx;
100
101 /* extern or resolved/global symbol */
102 bool is_extern;
103 /* weak or strong symbol, never goes back from strong to weak */
104 bool is_weak;
105 };
106
107 struct dst_sec {
108 char *sec_name;
109 /* positional (not necessarily ELF) index in an array of sections */
110 int id;
111
112 bool ephemeral;
113
114 /* ELF info */
115 size_t sec_idx;
116 Elf_Scn *scn;
117 Elf64_Shdr *shdr;
118 Elf_Data *data;
119
120 /* final output section size */
121 int sec_sz;
122 /* final output contents of the section */
123 void *raw_data;
124
125 /* corresponding STT_SECTION symbol index in SYMTAB */
126 int sec_sym_idx;
127
128 /* section's DATASEC variable info, emitted on BTF finalization */
129 bool has_btf;
130 int sec_var_cnt;
131 struct btf_var_secinfo *sec_vars;
132
133 /* section's .BTF.ext data */
134 struct btf_ext_sec_data func_info;
135 struct btf_ext_sec_data line_info;
136 struct btf_ext_sec_data core_relo_info;
137 };
138
139 struct bpf_linker {
140 char *filename;
141 int fd;
142 Elf *elf;
143 Elf64_Ehdr *elf_hdr;
144 bool swapped_endian;
145
146 /* Output sections metadata */
147 struct dst_sec *secs;
148 int sec_cnt;
149
150 struct strset *strtab_strs; /* STRTAB unique strings */
151 size_t strtab_sec_idx; /* STRTAB section index */
152 size_t symtab_sec_idx; /* SYMTAB section index */
153
154 struct btf *btf;
155 struct btf_ext *btf_ext;
156
157 /* global (including extern) ELF symbols */
158 int glob_sym_cnt;
159 struct glob_sym *glob_syms;
160
161 bool fd_is_owned;
162 };
163
164 #define pr_warn_elf(fmt, ...) \
165 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
166
167 static int init_output_elf(struct bpf_linker *linker);
168
169 static int bpf_linker_add_file(struct bpf_linker *linker, int fd,
170 const char *filename);
171
172 static int linker_load_obj_file(struct bpf_linker *linker,
173 struct src_obj *obj);
174 static int linker_sanity_check_elf(struct src_obj *obj);
175 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
176 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
177 static int linker_sanity_check_btf(struct src_obj *obj);
178 static int linker_sanity_check_btf_ext(struct src_obj *obj);
179 static int linker_fixup_btf(struct src_obj *obj);
180 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
181 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
182 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
183 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
184 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
185 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
186 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
187
188 static int finalize_btf(struct bpf_linker *linker);
189 static int finalize_btf_ext(struct bpf_linker *linker);
190
bpf_linker__free(struct bpf_linker * linker)191 void bpf_linker__free(struct bpf_linker *linker)
192 {
193 int i;
194
195 if (!linker)
196 return;
197
198 free(linker->filename);
199
200 if (linker->elf)
201 elf_end(linker->elf);
202
203 if (linker->fd >= 0 && linker->fd_is_owned)
204 close(linker->fd);
205
206 strset__free(linker->strtab_strs);
207
208 btf__free(linker->btf);
209 btf_ext__free(linker->btf_ext);
210
211 for (i = 1; i < linker->sec_cnt; i++) {
212 struct dst_sec *sec = &linker->secs[i];
213
214 free(sec->sec_name);
215 free(sec->raw_data);
216 free(sec->sec_vars);
217
218 free(sec->func_info.recs);
219 free(sec->line_info.recs);
220 free(sec->core_relo_info.recs);
221 }
222 free(linker->secs);
223
224 free(linker->glob_syms);
225 free(linker);
226 }
227
bpf_linker__new(const char * filename,struct bpf_linker_opts * opts)228 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
229 {
230 struct bpf_linker *linker;
231 int err;
232
233 if (!OPTS_VALID(opts, bpf_linker_opts))
234 return errno = EINVAL, NULL;
235
236 if (elf_version(EV_CURRENT) == EV_NONE) {
237 pr_warn_elf("libelf initialization failed");
238 return errno = EINVAL, NULL;
239 }
240
241 linker = calloc(1, sizeof(*linker));
242 if (!linker)
243 return errno = ENOMEM, NULL;
244
245 linker->filename = strdup(filename);
246 if (!linker->filename) {
247 err = -ENOMEM;
248 goto err_out;
249 }
250
251 linker->fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
252 if (linker->fd < 0) {
253 err = -errno;
254 pr_warn("failed to create '%s': %d\n", filename, err);
255 goto err_out;
256 }
257 linker->fd_is_owned = true;
258
259 err = init_output_elf(linker);
260 if (err)
261 goto err_out;
262
263 return linker;
264
265 err_out:
266 bpf_linker__free(linker);
267 return errno = -err, NULL;
268 }
269
bpf_linker__new_fd(int fd,struct bpf_linker_opts * opts)270 struct bpf_linker *bpf_linker__new_fd(int fd, struct bpf_linker_opts *opts)
271 {
272 struct bpf_linker *linker;
273 char filename[32];
274 int err;
275
276 if (fd < 0)
277 return errno = EINVAL, NULL;
278
279 if (!OPTS_VALID(opts, bpf_linker_opts))
280 return errno = EINVAL, NULL;
281
282 if (elf_version(EV_CURRENT) == EV_NONE) {
283 pr_warn_elf("libelf initialization failed");
284 return errno = EINVAL, NULL;
285 }
286
287 linker = calloc(1, sizeof(*linker));
288 if (!linker)
289 return errno = ENOMEM, NULL;
290
291 snprintf(filename, sizeof(filename), "fd:%d", fd);
292 linker->filename = strdup(filename);
293 if (!linker->filename) {
294 err = -ENOMEM;
295 goto err_out;
296 }
297
298 linker->fd = fd;
299 linker->fd_is_owned = false;
300
301 err = init_output_elf(linker);
302 if (err)
303 goto err_out;
304
305 return linker;
306
307 err_out:
308 bpf_linker__free(linker);
309 return errno = -err, NULL;
310 }
311
add_dst_sec(struct bpf_linker * linker,const char * sec_name)312 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
313 {
314 struct dst_sec *secs = linker->secs, *sec;
315 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
316
317 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
318 if (!secs)
319 return NULL;
320
321 /* zero out newly allocated memory */
322 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
323
324 linker->secs = secs;
325 linker->sec_cnt = new_cnt;
326
327 sec = &linker->secs[new_cnt - 1];
328 sec->id = new_cnt - 1;
329 sec->sec_name = strdup(sec_name);
330 if (!sec->sec_name)
331 return NULL;
332
333 return sec;
334 }
335
add_new_sym(struct bpf_linker * linker,size_t * sym_idx)336 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
337 {
338 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
339 Elf64_Sym *syms, *sym;
340 size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
341
342 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
343 if (!syms)
344 return NULL;
345
346 sym = &syms[sym_cnt];
347 memset(sym, 0, sizeof(*sym));
348
349 symtab->raw_data = syms;
350 symtab->sec_sz += sizeof(*sym);
351 symtab->shdr->sh_size += sizeof(*sym);
352 symtab->data->d_size += sizeof(*sym);
353
354 if (sym_idx)
355 *sym_idx = sym_cnt;
356
357 return sym;
358 }
359
init_output_elf(struct bpf_linker * linker)360 static int init_output_elf(struct bpf_linker *linker)
361 {
362 int err, str_off;
363 Elf64_Sym *init_sym;
364 struct dst_sec *sec;
365
366 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
367 if (!linker->elf) {
368 pr_warn_elf("failed to create ELF object");
369 return -EINVAL;
370 }
371
372 /* ELF header */
373 linker->elf_hdr = elf64_newehdr(linker->elf);
374 if (!linker->elf_hdr) {
375 pr_warn_elf("failed to create ELF header");
376 return -EINVAL;
377 }
378
379 linker->elf_hdr->e_machine = EM_BPF;
380 linker->elf_hdr->e_type = ET_REL;
381 /* Set unknown ELF endianness, assign later from input files */
382 linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE;
383
384 /* STRTAB */
385 /* initialize strset with an empty string to conform to ELF */
386 linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
387 if (libbpf_get_error(linker->strtab_strs))
388 return libbpf_get_error(linker->strtab_strs);
389
390 sec = add_dst_sec(linker, ".strtab");
391 if (!sec)
392 return -ENOMEM;
393
394 sec->scn = elf_newscn(linker->elf);
395 if (!sec->scn) {
396 pr_warn_elf("failed to create STRTAB section");
397 return -EINVAL;
398 }
399
400 sec->shdr = elf64_getshdr(sec->scn);
401 if (!sec->shdr)
402 return -EINVAL;
403
404 sec->data = elf_newdata(sec->scn);
405 if (!sec->data) {
406 pr_warn_elf("failed to create STRTAB data");
407 return -EINVAL;
408 }
409
410 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
411 if (str_off < 0)
412 return str_off;
413
414 sec->sec_idx = elf_ndxscn(sec->scn);
415 linker->elf_hdr->e_shstrndx = sec->sec_idx;
416 linker->strtab_sec_idx = sec->sec_idx;
417
418 sec->shdr->sh_name = str_off;
419 sec->shdr->sh_type = SHT_STRTAB;
420 sec->shdr->sh_flags = SHF_STRINGS;
421 sec->shdr->sh_offset = 0;
422 sec->shdr->sh_link = 0;
423 sec->shdr->sh_info = 0;
424 sec->shdr->sh_addralign = 1;
425 sec->shdr->sh_size = sec->sec_sz = 0;
426 sec->shdr->sh_entsize = 0;
427
428 /* SYMTAB */
429 sec = add_dst_sec(linker, ".symtab");
430 if (!sec)
431 return -ENOMEM;
432
433 sec->scn = elf_newscn(linker->elf);
434 if (!sec->scn) {
435 pr_warn_elf("failed to create SYMTAB section");
436 return -EINVAL;
437 }
438
439 sec->shdr = elf64_getshdr(sec->scn);
440 if (!sec->shdr)
441 return -EINVAL;
442
443 sec->data = elf_newdata(sec->scn);
444 if (!sec->data) {
445 pr_warn_elf("failed to create SYMTAB data");
446 return -EINVAL;
447 }
448 /* Ensure libelf translates byte-order of symbol records */
449 sec->data->d_type = ELF_T_SYM;
450
451 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
452 if (str_off < 0)
453 return str_off;
454
455 sec->sec_idx = elf_ndxscn(sec->scn);
456 linker->symtab_sec_idx = sec->sec_idx;
457
458 sec->shdr->sh_name = str_off;
459 sec->shdr->sh_type = SHT_SYMTAB;
460 sec->shdr->sh_flags = 0;
461 sec->shdr->sh_offset = 0;
462 sec->shdr->sh_link = linker->strtab_sec_idx;
463 /* sh_info should be one greater than the index of the last local
464 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
465 */
466 sec->shdr->sh_info = 0;
467 sec->shdr->sh_addralign = 8;
468 sec->shdr->sh_entsize = sizeof(Elf64_Sym);
469
470 /* .BTF */
471 linker->btf = btf__new_empty();
472 err = libbpf_get_error(linker->btf);
473 if (err)
474 return err;
475
476 /* add the special all-zero symbol */
477 init_sym = add_new_sym(linker, NULL);
478 if (!init_sym)
479 return -EINVAL;
480
481 init_sym->st_name = 0;
482 init_sym->st_info = 0;
483 init_sym->st_other = 0;
484 init_sym->st_shndx = SHN_UNDEF;
485 init_sym->st_value = 0;
486 init_sym->st_size = 0;
487
488 return 0;
489 }
490
bpf_linker_add_file(struct bpf_linker * linker,int fd,const char * filename)491 static int bpf_linker_add_file(struct bpf_linker *linker, int fd,
492 const char *filename)
493 {
494 struct src_obj obj = {};
495 int err = 0;
496
497 obj.filename = filename;
498 obj.fd = fd;
499
500 err = err ?: linker_load_obj_file(linker, &obj);
501 err = err ?: linker_append_sec_data(linker, &obj);
502 err = err ?: linker_append_elf_syms(linker, &obj);
503 err = err ?: linker_append_elf_relos(linker, &obj);
504 err = err ?: linker_append_btf(linker, &obj);
505 err = err ?: linker_append_btf_ext(linker, &obj);
506
507 /* free up src_obj resources */
508 free(obj.btf_type_map);
509 btf__free(obj.btf);
510 btf_ext__free(obj.btf_ext);
511 free(obj.secs);
512 free(obj.sym_map);
513 if (obj.elf)
514 elf_end(obj.elf);
515
516 return err;
517 }
518
bpf_linker__add_file(struct bpf_linker * linker,const char * filename,const struct bpf_linker_file_opts * opts)519 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
520 const struct bpf_linker_file_opts *opts)
521 {
522 int fd, err;
523
524 if (!OPTS_VALID(opts, bpf_linker_file_opts))
525 return libbpf_err(-EINVAL);
526
527 if (!linker->elf)
528 return libbpf_err(-EINVAL);
529
530 fd = open(filename, O_RDONLY | O_CLOEXEC);
531 if (fd < 0) {
532 err = -errno;
533 pr_warn("failed to open file '%s': %s\n", filename, errstr(err));
534 return libbpf_err(err);
535 }
536
537 err = bpf_linker_add_file(linker, fd, filename);
538 close(fd);
539 return libbpf_err(err);
540 }
541
bpf_linker__add_fd(struct bpf_linker * linker,int fd,const struct bpf_linker_file_opts * opts)542 int bpf_linker__add_fd(struct bpf_linker *linker, int fd,
543 const struct bpf_linker_file_opts *opts)
544 {
545 char filename[32];
546 int err;
547
548 if (!OPTS_VALID(opts, bpf_linker_file_opts))
549 return libbpf_err(-EINVAL);
550
551 if (!linker->elf)
552 return libbpf_err(-EINVAL);
553
554 if (fd < 0)
555 return libbpf_err(-EINVAL);
556
557 snprintf(filename, sizeof(filename), "fd:%d", fd);
558 err = bpf_linker_add_file(linker, fd, filename);
559 return libbpf_err(err);
560 }
561
bpf_linker__add_buf(struct bpf_linker * linker,void * buf,size_t buf_sz,const struct bpf_linker_file_opts * opts)562 int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz,
563 const struct bpf_linker_file_opts *opts)
564 {
565 char filename[32];
566 int fd, written, ret;
567
568 if (!OPTS_VALID(opts, bpf_linker_file_opts))
569 return libbpf_err(-EINVAL);
570
571 if (!linker->elf)
572 return libbpf_err(-EINVAL);
573
574 snprintf(filename, sizeof(filename), "mem:%p+%zu", buf, buf_sz);
575
576 fd = memfd_create(filename, 0);
577 if (fd < 0) {
578 ret = -errno;
579 pr_warn("failed to create memfd '%s': %s\n", filename, errstr(ret));
580 return libbpf_err(ret);
581 }
582
583 written = 0;
584 while (written < buf_sz) {
585 ret = write(fd, buf, buf_sz);
586 if (ret < 0) {
587 ret = -errno;
588 pr_warn("failed to write '%s': %s\n", filename, errstr(ret));
589 goto err_out;
590 }
591 written += ret;
592 }
593
594 ret = bpf_linker_add_file(linker, fd, filename);
595 err_out:
596 close(fd);
597 return libbpf_err(ret);
598 }
599
is_dwarf_sec_name(const char * name)600 static bool is_dwarf_sec_name(const char *name)
601 {
602 /* approximation, but the actual list is too long */
603 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
604 }
605
is_ignored_sec(struct src_sec * sec)606 static bool is_ignored_sec(struct src_sec *sec)
607 {
608 Elf64_Shdr *shdr = sec->shdr;
609 const char *name = sec->sec_name;
610
611 /* no special handling of .strtab */
612 if (shdr->sh_type == SHT_STRTAB)
613 return true;
614
615 /* ignore .llvm_addrsig section as well */
616 if (shdr->sh_type == SHT_LLVM_ADDRSIG)
617 return true;
618
619 /* no subprograms will lead to an empty .text section, ignore it */
620 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
621 strcmp(sec->sec_name, ".text") == 0)
622 return true;
623
624 /* DWARF sections */
625 if (is_dwarf_sec_name(sec->sec_name))
626 return true;
627
628 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
629 name += sizeof(".rel") - 1;
630 /* DWARF section relocations */
631 if (is_dwarf_sec_name(name))
632 return true;
633
634 /* .BTF and .BTF.ext don't need relocations */
635 if (strcmp(name, BTF_ELF_SEC) == 0 ||
636 strcmp(name, BTF_EXT_ELF_SEC) == 0)
637 return true;
638 }
639
640 return false;
641 }
642
add_src_sec(struct src_obj * obj,const char * sec_name)643 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
644 {
645 struct src_sec *secs = obj->secs, *sec;
646 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
647
648 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
649 if (!secs)
650 return NULL;
651
652 /* zero out newly allocated memory */
653 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
654
655 obj->secs = secs;
656 obj->sec_cnt = new_cnt;
657
658 sec = &obj->secs[new_cnt - 1];
659 sec->id = new_cnt - 1;
660 sec->sec_name = sec_name;
661
662 return sec;
663 }
664
linker_load_obj_file(struct bpf_linker * linker,struct src_obj * obj)665 static int linker_load_obj_file(struct bpf_linker *linker,
666 struct src_obj *obj)
667 {
668 int err = 0;
669 Elf_Scn *scn;
670 Elf_Data *data;
671 Elf64_Ehdr *ehdr;
672 Elf64_Shdr *shdr;
673 struct src_sec *sec;
674 unsigned char obj_byteorder;
675 unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA];
676 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
677 const unsigned char host_byteorder = ELFDATA2LSB;
678 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
679 const unsigned char host_byteorder = ELFDATA2MSB;
680 #else
681 #error "Unknown __BYTE_ORDER__"
682 #endif
683
684 pr_debug("linker: adding object file '%s'...\n", obj->filename);
685
686 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
687 if (!obj->elf) {
688 pr_warn_elf("failed to parse ELF file '%s'", obj->filename);
689 return -EINVAL;
690 }
691
692 /* Sanity check ELF file high-level properties */
693 ehdr = elf64_getehdr(obj->elf);
694 if (!ehdr) {
695 pr_warn_elf("failed to get ELF header for %s", obj->filename);
696 return -EINVAL;
697 }
698
699 /* Linker output endianness set by first input object */
700 obj_byteorder = ehdr->e_ident[EI_DATA];
701 if (obj_byteorder != ELFDATA2LSB && obj_byteorder != ELFDATA2MSB) {
702 err = -EOPNOTSUPP;
703 pr_warn("unknown byte order of ELF file %s\n", obj->filename);
704 return err;
705 }
706 if (link_byteorder == ELFDATANONE) {
707 linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder;
708 linker->swapped_endian = obj_byteorder != host_byteorder;
709 pr_debug("linker: set %s-endian output byte order\n",
710 obj_byteorder == ELFDATA2MSB ? "big" : "little");
711 } else if (link_byteorder != obj_byteorder) {
712 err = -EOPNOTSUPP;
713 pr_warn("byte order mismatch with ELF file %s\n", obj->filename);
714 return err;
715 }
716
717 if (ehdr->e_type != ET_REL
718 || ehdr->e_machine != EM_BPF
719 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
720 err = -EOPNOTSUPP;
721 pr_warn_elf("unsupported kind of ELF file %s", obj->filename);
722 return err;
723 }
724
725 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
726 pr_warn_elf("failed to get SHSTRTAB section index for %s", obj->filename);
727 return -EINVAL;
728 }
729
730 scn = NULL;
731 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
732 size_t sec_idx = elf_ndxscn(scn);
733 const char *sec_name;
734
735 shdr = elf64_getshdr(scn);
736 if (!shdr) {
737 pr_warn_elf("failed to get section #%zu header for %s",
738 sec_idx, obj->filename);
739 return -EINVAL;
740 }
741
742 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
743 if (!sec_name) {
744 pr_warn_elf("failed to get section #%zu name for %s",
745 sec_idx, obj->filename);
746 return -EINVAL;
747 }
748
749 data = elf_getdata(scn, 0);
750 if (!data) {
751 pr_warn_elf("failed to get section #%zu (%s) data from %s",
752 sec_idx, sec_name, obj->filename);
753 return -EINVAL;
754 }
755
756 sec = add_src_sec(obj, sec_name);
757 if (!sec)
758 return -ENOMEM;
759
760 sec->scn = scn;
761 sec->shdr = shdr;
762 sec->data = data;
763 sec->sec_idx = elf_ndxscn(scn);
764
765 if (is_ignored_sec(sec)) {
766 sec->skipped = true;
767 continue;
768 }
769
770 switch (shdr->sh_type) {
771 case SHT_SYMTAB:
772 if (obj->symtab_sec_idx) {
773 err = -EOPNOTSUPP;
774 pr_warn("multiple SYMTAB sections found, not supported\n");
775 return err;
776 }
777 obj->symtab_sec_idx = sec_idx;
778 break;
779 case SHT_STRTAB:
780 /* we'll construct our own string table */
781 break;
782 case SHT_PROGBITS:
783 if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
784 obj->btf = btf__new(data->d_buf, shdr->sh_size);
785 err = libbpf_get_error(obj->btf);
786 if (err) {
787 pr_warn("failed to parse .BTF from %s: %s\n",
788 obj->filename, errstr(err));
789 return err;
790 }
791 sec->skipped = true;
792 continue;
793 }
794 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
795 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
796 err = libbpf_get_error(obj->btf_ext);
797 if (err) {
798 pr_warn("failed to parse .BTF.ext from '%s': %s\n",
799 obj->filename, errstr(err));
800 return err;
801 }
802 sec->skipped = true;
803 continue;
804 }
805
806 /* data & code */
807 break;
808 case SHT_NOBITS:
809 /* BSS */
810 break;
811 case SHT_REL:
812 /* relocations */
813 break;
814 default:
815 pr_warn("unrecognized section #%zu (%s) in %s\n",
816 sec_idx, sec_name, obj->filename);
817 err = -EINVAL;
818 return err;
819 }
820 }
821
822 err = err ?: linker_sanity_check_elf(obj);
823 err = err ?: linker_sanity_check_btf(obj);
824 err = err ?: linker_sanity_check_btf_ext(obj);
825 err = err ?: linker_fixup_btf(obj);
826
827 return err;
828 }
829
linker_sanity_check_elf(struct src_obj * obj)830 static int linker_sanity_check_elf(struct src_obj *obj)
831 {
832 struct src_sec *sec;
833 int i, err;
834
835 if (!obj->symtab_sec_idx) {
836 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
837 return -EINVAL;
838 }
839 if (!obj->shstrs_sec_idx) {
840 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
841 return -EINVAL;
842 }
843
844 for (i = 1; i < obj->sec_cnt; i++) {
845 sec = &obj->secs[i];
846
847 if (sec->sec_name[0] == '\0') {
848 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
849 return -EINVAL;
850 }
851
852 if (is_dwarf_sec_name(sec->sec_name))
853 continue;
854
855 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {
856 pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",
857 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
858 obj->filename);
859 return -EINVAL;
860 }
861 if (sec->shdr->sh_addralign != sec->data->d_align) {
862 pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",
863 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
864 (long long unsigned)sec->data->d_align, obj->filename);
865 return -EINVAL;
866 }
867
868 if (sec->shdr->sh_size != sec->data->d_size) {
869 pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",
870 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
871 (long long unsigned)sec->data->d_size, obj->filename);
872 return -EINVAL;
873 }
874
875 switch (sec->shdr->sh_type) {
876 case SHT_SYMTAB:
877 err = linker_sanity_check_elf_symtab(obj, sec);
878 if (err)
879 return err;
880 break;
881 case SHT_STRTAB:
882 break;
883 case SHT_PROGBITS:
884 if (sec->shdr->sh_flags & SHF_EXECINSTR) {
885 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {
886 pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",
887 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
888 obj->filename);
889 return -EINVAL;
890 }
891 }
892 break;
893 case SHT_NOBITS:
894 break;
895 case SHT_REL:
896 err = linker_sanity_check_elf_relos(obj, sec);
897 if (err)
898 return err;
899 break;
900 case SHT_LLVM_ADDRSIG:
901 break;
902 default:
903 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
904 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
905 return -EINVAL;
906 }
907 }
908
909 return 0;
910 }
911
linker_sanity_check_elf_symtab(struct src_obj * obj,struct src_sec * sec)912 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
913 {
914 struct src_sec *link_sec;
915 Elf64_Sym *sym;
916 int i, n;
917
918 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
919 return -EINVAL;
920 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
921 return -EINVAL;
922
923 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
924 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
925 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
926 return -EINVAL;
927 }
928 link_sec = &obj->secs[sec->shdr->sh_link];
929 if (link_sec->shdr->sh_type != SHT_STRTAB) {
930 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
931 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
932 return -EINVAL;
933 }
934
935 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
936 sym = sec->data->d_buf;
937 for (i = 0; i < n; i++, sym++) {
938 int sym_type = ELF64_ST_TYPE(sym->st_info);
939 int sym_bind = ELF64_ST_BIND(sym->st_info);
940 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
941
942 if (i == 0) {
943 if (sym->st_name != 0 || sym->st_info != 0
944 || sym->st_other != 0 || sym->st_shndx != 0
945 || sym->st_value != 0 || sym->st_size != 0) {
946 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
947 return -EINVAL;
948 }
949 continue;
950 }
951 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
952 pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
953 i, sec->sec_idx, sym_bind);
954 return -EINVAL;
955 }
956 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
957 pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
958 i, sec->sec_idx, sym_vis);
959 return -EINVAL;
960 }
961 if (sym->st_shndx == 0) {
962 if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
963 || sym->st_value != 0 || sym->st_size != 0) {
964 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
965 i, obj->filename);
966
967 return -EINVAL;
968 }
969 continue;
970 }
971 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
972 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
973 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
974 return -EINVAL;
975 }
976 if (sym_type == STT_SECTION) {
977 if (sym->st_value != 0)
978 return -EINVAL;
979 continue;
980 }
981 }
982
983 return 0;
984 }
985
linker_sanity_check_elf_relos(struct src_obj * obj,struct src_sec * sec)986 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
987 {
988 struct src_sec *link_sec, *sym_sec;
989 Elf64_Rel *relo;
990 int i, n;
991
992 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
993 return -EINVAL;
994 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
995 return -EINVAL;
996
997 /* SHT_REL's sh_link should point to SYMTAB */
998 if (sec->shdr->sh_link != obj->symtab_sec_idx) {
999 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
1000 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
1001 return -EINVAL;
1002 }
1003
1004 /* SHT_REL's sh_info points to relocated section */
1005 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
1006 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
1007 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
1008 return -EINVAL;
1009 }
1010 link_sec = &obj->secs[sec->shdr->sh_info];
1011
1012 /* .rel<secname> -> <secname> pattern is followed */
1013 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
1014 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
1015 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
1016 sec->sec_idx, obj->filename);
1017 return -EINVAL;
1018 }
1019
1020 /* don't further validate relocations for ignored sections */
1021 if (link_sec->skipped)
1022 return 0;
1023
1024 /* relocatable section is data or instructions */
1025 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
1026 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
1027 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
1028 return -EINVAL;
1029 }
1030
1031 /* check sanity of each relocation */
1032 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
1033 relo = sec->data->d_buf;
1034 sym_sec = &obj->secs[obj->symtab_sec_idx];
1035 for (i = 0; i < n; i++, relo++) {
1036 size_t sym_idx = ELF64_R_SYM(relo->r_info);
1037 size_t sym_type = ELF64_R_TYPE(relo->r_info);
1038
1039 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
1040 sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
1041 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
1042 i, sec->sec_idx, sym_type, obj->filename);
1043 return -EINVAL;
1044 }
1045
1046 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
1047 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
1048 i, sec->sec_idx, sym_idx, obj->filename);
1049 return -EINVAL;
1050 }
1051
1052 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
1053 if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
1054 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
1055 i, sec->sec_idx, sym_idx, obj->filename);
1056 return -EINVAL;
1057 }
1058 }
1059 }
1060
1061 return 0;
1062 }
1063
check_btf_type_id(__u32 * type_id,void * ctx)1064 static int check_btf_type_id(__u32 *type_id, void *ctx)
1065 {
1066 struct btf *btf = ctx;
1067
1068 if (*type_id >= btf__type_cnt(btf))
1069 return -EINVAL;
1070
1071 return 0;
1072 }
1073
check_btf_str_off(__u32 * str_off,void * ctx)1074 static int check_btf_str_off(__u32 *str_off, void *ctx)
1075 {
1076 struct btf *btf = ctx;
1077 const char *s;
1078
1079 s = btf__str_by_offset(btf, *str_off);
1080
1081 if (!s)
1082 return -EINVAL;
1083
1084 return 0;
1085 }
1086
linker_sanity_check_btf(struct src_obj * obj)1087 static int linker_sanity_check_btf(struct src_obj *obj)
1088 {
1089 struct btf_type *t;
1090 int i, n, err;
1091
1092 if (!obj->btf)
1093 return 0;
1094
1095 n = btf__type_cnt(obj->btf);
1096 for (i = 1; i < n; i++) {
1097 struct btf_field_iter it;
1098 __u32 *type_id, *str_off;
1099
1100 t = btf_type_by_id(obj->btf, i);
1101
1102 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
1103 if (err)
1104 return err;
1105 while ((type_id = btf_field_iter_next(&it))) {
1106 if (*type_id >= n)
1107 return -EINVAL;
1108 }
1109
1110 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
1111 if (err)
1112 return err;
1113 while ((str_off = btf_field_iter_next(&it))) {
1114 if (!btf__str_by_offset(obj->btf, *str_off))
1115 return -EINVAL;
1116 }
1117 }
1118
1119 return 0;
1120 }
1121
linker_sanity_check_btf_ext(struct src_obj * obj)1122 static int linker_sanity_check_btf_ext(struct src_obj *obj)
1123 {
1124 int err = 0;
1125
1126 if (!obj->btf_ext)
1127 return 0;
1128
1129 /* can't use .BTF.ext without .BTF */
1130 if (!obj->btf)
1131 return -EINVAL;
1132
1133 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
1134 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
1135 if (err)
1136 return err;
1137
1138 return 0;
1139 }
1140
init_sec(struct bpf_linker * linker,struct dst_sec * dst_sec,struct src_sec * src_sec)1141 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
1142 {
1143 Elf_Scn *scn;
1144 Elf_Data *data;
1145 Elf64_Shdr *shdr;
1146 int name_off;
1147
1148 dst_sec->sec_sz = 0;
1149 dst_sec->sec_idx = 0;
1150 dst_sec->ephemeral = src_sec->ephemeral;
1151
1152 /* ephemeral sections are just thin section shells lacking most parts */
1153 if (src_sec->ephemeral)
1154 return 0;
1155
1156 scn = elf_newscn(linker->elf);
1157 if (!scn)
1158 return -ENOMEM;
1159 data = elf_newdata(scn);
1160 if (!data)
1161 return -ENOMEM;
1162 shdr = elf64_getshdr(scn);
1163 if (!shdr)
1164 return -ENOMEM;
1165
1166 dst_sec->scn = scn;
1167 dst_sec->shdr = shdr;
1168 dst_sec->data = data;
1169 dst_sec->sec_idx = elf_ndxscn(scn);
1170
1171 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1172 if (name_off < 0)
1173 return name_off;
1174
1175 shdr->sh_name = name_off;
1176 shdr->sh_type = src_sec->shdr->sh_type;
1177 shdr->sh_flags = src_sec->shdr->sh_flags;
1178 shdr->sh_size = 0;
1179 /* sh_link and sh_info have different meaning for different types of
1180 * sections, so we leave it up to the caller code to fill them in, if
1181 * necessary
1182 */
1183 shdr->sh_link = 0;
1184 shdr->sh_info = 0;
1185 shdr->sh_addralign = src_sec->shdr->sh_addralign;
1186 shdr->sh_entsize = src_sec->shdr->sh_entsize;
1187
1188 data->d_type = src_sec->data->d_type;
1189 data->d_size = 0;
1190 data->d_buf = NULL;
1191 data->d_align = src_sec->data->d_align;
1192 data->d_off = 0;
1193
1194 return 0;
1195 }
1196
find_dst_sec_by_name(struct bpf_linker * linker,const char * sec_name)1197 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1198 {
1199 struct dst_sec *sec;
1200 int i;
1201
1202 for (i = 1; i < linker->sec_cnt; i++) {
1203 sec = &linker->secs[i];
1204
1205 if (strcmp(sec->sec_name, sec_name) == 0)
1206 return sec;
1207 }
1208
1209 return NULL;
1210 }
1211
secs_match(struct dst_sec * dst,struct src_sec * src)1212 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1213 {
1214 if (dst->ephemeral || src->ephemeral)
1215 return true;
1216
1217 if (dst->shdr->sh_type != src->shdr->sh_type) {
1218 pr_warn("sec %s types mismatch\n", dst->sec_name);
1219 return false;
1220 }
1221 if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1222 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1223 return false;
1224 }
1225 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1226 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1227 return false;
1228 }
1229
1230 return true;
1231 }
1232
sec_content_is_same(struct dst_sec * dst_sec,struct src_sec * src_sec)1233 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1234 {
1235 if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1236 return false;
1237 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1238 return false;
1239 return true;
1240 }
1241
is_exec_sec(struct dst_sec * sec)1242 static bool is_exec_sec(struct dst_sec *sec)
1243 {
1244 if (!sec || sec->ephemeral)
1245 return false;
1246 return (sec->shdr->sh_type == SHT_PROGBITS) &&
1247 (sec->shdr->sh_flags & SHF_EXECINSTR);
1248 }
1249
exec_sec_bswap(void * raw_data,int size)1250 static void exec_sec_bswap(void *raw_data, int size)
1251 {
1252 const int insn_cnt = size / sizeof(struct bpf_insn);
1253 struct bpf_insn *insn = raw_data;
1254 int i;
1255
1256 for (i = 0; i < insn_cnt; i++, insn++)
1257 bpf_insn_bswap(insn);
1258 }
1259
extend_sec(struct bpf_linker * linker,struct dst_sec * dst,struct src_sec * src)1260 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1261 {
1262 void *tmp;
1263 size_t dst_align, src_align;
1264 size_t dst_align_sz, dst_final_sz;
1265 int err;
1266
1267 /* Ephemeral source section doesn't contribute anything to ELF
1268 * section data.
1269 */
1270 if (src->ephemeral)
1271 return 0;
1272
1273 /* Some sections (like .maps) can contain both externs (and thus be
1274 * ephemeral) and non-externs (map definitions). So it's possible that
1275 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1276 * first non-ephemeral entity appears. In such case, we add ELF
1277 * section, data, etc.
1278 */
1279 if (dst->ephemeral) {
1280 err = init_sec(linker, dst, src);
1281 if (err)
1282 return err;
1283 }
1284
1285 dst_align = dst->shdr->sh_addralign;
1286 src_align = src->shdr->sh_addralign;
1287 if (dst_align == 0)
1288 dst_align = 1;
1289 if (dst_align < src_align)
1290 dst_align = src_align;
1291
1292 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1293
1294 /* no need to re-align final size */
1295 dst_final_sz = dst_align_sz + src->shdr->sh_size;
1296
1297 if (src->shdr->sh_type != SHT_NOBITS) {
1298 tmp = realloc(dst->raw_data, dst_final_sz);
1299 /* If dst_align_sz == 0, realloc() behaves in a special way:
1300 * 1. When dst->raw_data is NULL it returns:
1301 * "either NULL or a pointer suitable to be passed to free()" [1].
1302 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1303 * thus invalidating any "pointer suitable to be passed to free()" obtained
1304 * at step (1).
1305 *
1306 * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1307 * dst->raw_data would be freed again in bpf_linker__free().
1308 *
1309 * [1] man 3 realloc
1310 */
1311 if (!tmp && dst_align_sz > 0)
1312 return -ENOMEM;
1313 dst->raw_data = tmp;
1314
1315 /* pad dst section, if it's alignment forced size increase */
1316 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1317 /* now copy src data at a properly aligned offset */
1318 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1319
1320 /* convert added bpf insns to native byte-order */
1321 if (linker->swapped_endian && is_exec_sec(dst))
1322 exec_sec_bswap(dst->raw_data + dst_align_sz, src->shdr->sh_size);
1323 }
1324
1325 dst->sec_sz = dst_final_sz;
1326 dst->shdr->sh_size = dst_final_sz;
1327 dst->data->d_size = dst_final_sz;
1328
1329 dst->shdr->sh_addralign = dst_align;
1330 dst->data->d_align = dst_align;
1331
1332 src->dst_off = dst_align_sz;
1333
1334 return 0;
1335 }
1336
is_data_sec(struct src_sec * sec)1337 static bool is_data_sec(struct src_sec *sec)
1338 {
1339 if (!sec || sec->skipped)
1340 return false;
1341 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1342 if (sec->ephemeral)
1343 return true;
1344 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1345 }
1346
is_relo_sec(struct src_sec * sec)1347 static bool is_relo_sec(struct src_sec *sec)
1348 {
1349 if (!sec || sec->skipped || sec->ephemeral)
1350 return false;
1351 return sec->shdr->sh_type == SHT_REL;
1352 }
1353
linker_append_sec_data(struct bpf_linker * linker,struct src_obj * obj)1354 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1355 {
1356 int i, err;
1357
1358 for (i = 1; i < obj->sec_cnt; i++) {
1359 struct src_sec *src_sec;
1360 struct dst_sec *dst_sec;
1361
1362 src_sec = &obj->secs[i];
1363 if (!is_data_sec(src_sec))
1364 continue;
1365
1366 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1367 if (!dst_sec) {
1368 dst_sec = add_dst_sec(linker, src_sec->sec_name);
1369 if (!dst_sec)
1370 return -ENOMEM;
1371 err = init_sec(linker, dst_sec, src_sec);
1372 if (err) {
1373 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1374 return err;
1375 }
1376 } else {
1377 if (!secs_match(dst_sec, src_sec)) {
1378 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1379 return -1;
1380 }
1381
1382 /* "license" and "version" sections are deduped */
1383 if (strcmp(src_sec->sec_name, "license") == 0
1384 || strcmp(src_sec->sec_name, "version") == 0) {
1385 if (!sec_content_is_same(dst_sec, src_sec)) {
1386 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1387 return -EINVAL;
1388 }
1389 src_sec->skipped = true;
1390 src_sec->dst_id = dst_sec->id;
1391 continue;
1392 }
1393 }
1394
1395 /* record mapped section index */
1396 src_sec->dst_id = dst_sec->id;
1397
1398 err = extend_sec(linker, dst_sec, src_sec);
1399 if (err)
1400 return err;
1401 }
1402
1403 return 0;
1404 }
1405
linker_append_elf_syms(struct bpf_linker * linker,struct src_obj * obj)1406 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1407 {
1408 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1409 Elf64_Sym *sym = symtab->data->d_buf;
1410 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1411 int str_sec_idx = symtab->shdr->sh_link;
1412 const char *sym_name;
1413
1414 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1415 if (!obj->sym_map)
1416 return -ENOMEM;
1417
1418 for (i = 0; i < n; i++, sym++) {
1419 /* We already validated all-zero symbol #0 and we already
1420 * appended it preventively to the final SYMTAB, so skip it.
1421 */
1422 if (i == 0)
1423 continue;
1424
1425 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1426 if (!sym_name) {
1427 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1428 return -EINVAL;
1429 }
1430
1431 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1432 if (err)
1433 return err;
1434 }
1435
1436 return 0;
1437 }
1438
get_sym_by_idx(struct bpf_linker * linker,size_t sym_idx)1439 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1440 {
1441 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1442 Elf64_Sym *syms = symtab->raw_data;
1443
1444 return &syms[sym_idx];
1445 }
1446
find_glob_sym(struct bpf_linker * linker,const char * sym_name)1447 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1448 {
1449 struct glob_sym *glob_sym;
1450 const char *name;
1451 int i;
1452
1453 for (i = 0; i < linker->glob_sym_cnt; i++) {
1454 glob_sym = &linker->glob_syms[i];
1455 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1456
1457 if (strcmp(name, sym_name) == 0)
1458 return glob_sym;
1459 }
1460
1461 return NULL;
1462 }
1463
add_glob_sym(struct bpf_linker * linker)1464 static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1465 {
1466 struct glob_sym *syms, *sym;
1467
1468 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1469 sizeof(*linker->glob_syms));
1470 if (!syms)
1471 return NULL;
1472
1473 sym = &syms[linker->glob_sym_cnt];
1474 memset(sym, 0, sizeof(*sym));
1475 sym->var_idx = -1;
1476
1477 linker->glob_syms = syms;
1478 linker->glob_sym_cnt++;
1479
1480 return sym;
1481 }
1482
glob_sym_btf_matches(const char * sym_name,bool exact,const struct btf * btf1,__u32 id1,const struct btf * btf2,__u32 id2)1483 static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1484 const struct btf *btf1, __u32 id1,
1485 const struct btf *btf2, __u32 id2)
1486 {
1487 const struct btf_type *t1, *t2;
1488 bool is_static1, is_static2;
1489 const char *n1, *n2;
1490 int i, n;
1491
1492 recur:
1493 n1 = n2 = NULL;
1494 t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1495 t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1496
1497 /* check if only one side is FWD, otherwise handle with common logic */
1498 if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1499 n1 = btf__str_by_offset(btf1, t1->name_off);
1500 n2 = btf__str_by_offset(btf2, t2->name_off);
1501 if (strcmp(n1, n2) != 0) {
1502 pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1503 sym_name, n1, n2);
1504 return false;
1505 }
1506 /* validate if FWD kind matches concrete kind */
1507 if (btf_is_fwd(t1)) {
1508 if (btf_kflag(t1) && btf_is_union(t2))
1509 return true;
1510 if (!btf_kflag(t1) && btf_is_struct(t2))
1511 return true;
1512 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1513 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1514 } else {
1515 if (btf_kflag(t2) && btf_is_union(t1))
1516 return true;
1517 if (!btf_kflag(t2) && btf_is_struct(t1))
1518 return true;
1519 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1520 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1521 }
1522 return false;
1523 }
1524
1525 if (btf_kind(t1) != btf_kind(t2)) {
1526 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1527 sym_name, btf_kind_str(t1), btf_kind_str(t2));
1528 return false;
1529 }
1530
1531 switch (btf_kind(t1)) {
1532 case BTF_KIND_STRUCT:
1533 case BTF_KIND_UNION:
1534 case BTF_KIND_ENUM:
1535 case BTF_KIND_ENUM64:
1536 case BTF_KIND_FWD:
1537 case BTF_KIND_FUNC:
1538 case BTF_KIND_VAR:
1539 n1 = btf__str_by_offset(btf1, t1->name_off);
1540 n2 = btf__str_by_offset(btf2, t2->name_off);
1541 if (strcmp(n1, n2) != 0) {
1542 pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1543 sym_name, btf_kind_str(t1), n1, n2);
1544 return false;
1545 }
1546 break;
1547 default:
1548 break;
1549 }
1550
1551 switch (btf_kind(t1)) {
1552 case BTF_KIND_UNKN: /* void */
1553 case BTF_KIND_FWD:
1554 return true;
1555 case BTF_KIND_INT:
1556 case BTF_KIND_FLOAT:
1557 case BTF_KIND_ENUM:
1558 case BTF_KIND_ENUM64:
1559 /* ignore encoding for int and enum values for enum */
1560 if (t1->size != t2->size) {
1561 pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1562 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1563 return false;
1564 }
1565 return true;
1566 case BTF_KIND_PTR:
1567 /* just validate overall shape of the referenced type, so no
1568 * contents comparison for struct/union, and allowed fwd vs
1569 * struct/union
1570 */
1571 exact = false;
1572 id1 = t1->type;
1573 id2 = t2->type;
1574 goto recur;
1575 case BTF_KIND_ARRAY:
1576 /* ignore index type and array size */
1577 id1 = btf_array(t1)->type;
1578 id2 = btf_array(t2)->type;
1579 goto recur;
1580 case BTF_KIND_FUNC:
1581 /* extern and global linkages are compatible */
1582 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1583 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1584 if (is_static1 != is_static2) {
1585 pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1586 return false;
1587 }
1588
1589 id1 = t1->type;
1590 id2 = t2->type;
1591 goto recur;
1592 case BTF_KIND_VAR:
1593 /* extern and global linkages are compatible */
1594 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1595 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1596 if (is_static1 != is_static2) {
1597 pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1598 return false;
1599 }
1600
1601 id1 = t1->type;
1602 id2 = t2->type;
1603 goto recur;
1604 case BTF_KIND_STRUCT:
1605 case BTF_KIND_UNION: {
1606 const struct btf_member *m1, *m2;
1607
1608 if (!exact)
1609 return true;
1610
1611 if (btf_vlen(t1) != btf_vlen(t2)) {
1612 pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1613 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1614 return false;
1615 }
1616
1617 n = btf_vlen(t1);
1618 m1 = btf_members(t1);
1619 m2 = btf_members(t2);
1620 for (i = 0; i < n; i++, m1++, m2++) {
1621 n1 = btf__str_by_offset(btf1, m1->name_off);
1622 n2 = btf__str_by_offset(btf2, m2->name_off);
1623 if (strcmp(n1, n2) != 0) {
1624 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1625 sym_name, i, n1, n2);
1626 return false;
1627 }
1628 if (m1->offset != m2->offset) {
1629 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1630 sym_name, i, n1);
1631 return false;
1632 }
1633 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1634 return false;
1635 }
1636
1637 return true;
1638 }
1639 case BTF_KIND_FUNC_PROTO: {
1640 const struct btf_param *m1, *m2;
1641
1642 if (btf_vlen(t1) != btf_vlen(t2)) {
1643 pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1644 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1645 return false;
1646 }
1647
1648 n = btf_vlen(t1);
1649 m1 = btf_params(t1);
1650 m2 = btf_params(t2);
1651 for (i = 0; i < n; i++, m1++, m2++) {
1652 /* ignore func arg names */
1653 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1654 return false;
1655 }
1656
1657 /* now check return type as well */
1658 id1 = t1->type;
1659 id2 = t2->type;
1660 goto recur;
1661 }
1662
1663 /* skip_mods_and_typedefs() make this impossible */
1664 case BTF_KIND_TYPEDEF:
1665 case BTF_KIND_VOLATILE:
1666 case BTF_KIND_CONST:
1667 case BTF_KIND_RESTRICT:
1668 /* DATASECs are never compared with each other */
1669 case BTF_KIND_DATASEC:
1670 default:
1671 pr_warn("global '%s': unsupported BTF kind %s\n",
1672 sym_name, btf_kind_str(t1));
1673 return false;
1674 }
1675 }
1676
map_defs_match(const char * sym_name,const struct btf * main_btf,const struct btf_map_def * main_def,const struct btf_map_def * main_inner_def,const struct btf * extra_btf,const struct btf_map_def * extra_def,const struct btf_map_def * extra_inner_def)1677 static bool map_defs_match(const char *sym_name,
1678 const struct btf *main_btf,
1679 const struct btf_map_def *main_def,
1680 const struct btf_map_def *main_inner_def,
1681 const struct btf *extra_btf,
1682 const struct btf_map_def *extra_def,
1683 const struct btf_map_def *extra_inner_def)
1684 {
1685 const char *reason;
1686
1687 if (main_def->map_type != extra_def->map_type) {
1688 reason = "type";
1689 goto mismatch;
1690 }
1691
1692 /* check key type/size match */
1693 if (main_def->key_size != extra_def->key_size) {
1694 reason = "key_size";
1695 goto mismatch;
1696 }
1697 if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1698 reason = "key type";
1699 goto mismatch;
1700 }
1701 if ((main_def->parts & MAP_DEF_KEY_TYPE)
1702 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1703 main_btf, main_def->key_type_id,
1704 extra_btf, extra_def->key_type_id)) {
1705 reason = "key type";
1706 goto mismatch;
1707 }
1708
1709 /* validate value type/size match */
1710 if (main_def->value_size != extra_def->value_size) {
1711 reason = "value_size";
1712 goto mismatch;
1713 }
1714 if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1715 reason = "value type";
1716 goto mismatch;
1717 }
1718 if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1719 && !glob_sym_btf_matches(sym_name, true /*exact*/,
1720 main_btf, main_def->value_type_id,
1721 extra_btf, extra_def->value_type_id)) {
1722 reason = "key type";
1723 goto mismatch;
1724 }
1725
1726 if (main_def->max_entries != extra_def->max_entries) {
1727 reason = "max_entries";
1728 goto mismatch;
1729 }
1730 if (main_def->map_flags != extra_def->map_flags) {
1731 reason = "map_flags";
1732 goto mismatch;
1733 }
1734 if (main_def->numa_node != extra_def->numa_node) {
1735 reason = "numa_node";
1736 goto mismatch;
1737 }
1738 if (main_def->pinning != extra_def->pinning) {
1739 reason = "pinning";
1740 goto mismatch;
1741 }
1742
1743 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1744 reason = "inner map";
1745 goto mismatch;
1746 }
1747
1748 if (main_def->parts & MAP_DEF_INNER_MAP) {
1749 char inner_map_name[128];
1750
1751 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1752
1753 return map_defs_match(inner_map_name,
1754 main_btf, main_inner_def, NULL,
1755 extra_btf, extra_inner_def, NULL);
1756 }
1757
1758 return true;
1759
1760 mismatch:
1761 pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1762 return false;
1763 }
1764
glob_map_defs_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,int btf_id)1765 static bool glob_map_defs_match(const char *sym_name,
1766 struct bpf_linker *linker, struct glob_sym *glob_sym,
1767 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1768 {
1769 struct btf_map_def dst_def = {}, dst_inner_def = {};
1770 struct btf_map_def src_def = {}, src_inner_def = {};
1771 const struct btf_type *t;
1772 int err;
1773
1774 t = btf__type_by_id(obj->btf, btf_id);
1775 if (!btf_is_var(t)) {
1776 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1777 return false;
1778 }
1779 t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1780
1781 err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1782 if (err) {
1783 pr_warn("global '%s': invalid map definition\n", sym_name);
1784 return false;
1785 }
1786
1787 /* re-parse existing map definition */
1788 t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1789 t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1790 err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1791 if (err) {
1792 /* this should not happen, because we already validated it */
1793 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1794 return false;
1795 }
1796
1797 /* Currently extern map definition has to be complete and match
1798 * concrete map definition exactly. This restriction might be lifted
1799 * in the future.
1800 */
1801 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1802 obj->btf, &src_def, &src_inner_def);
1803 }
1804
glob_syms_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,size_t sym_idx,int btf_id)1805 static bool glob_syms_match(const char *sym_name,
1806 struct bpf_linker *linker, struct glob_sym *glob_sym,
1807 struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1808 {
1809 const struct btf_type *src_t;
1810
1811 /* if we are dealing with externs, BTF types describing both global
1812 * and extern VARs/FUNCs should be completely present in all files
1813 */
1814 if (!glob_sym->btf_id || !btf_id) {
1815 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1816 return false;
1817 }
1818
1819 src_t = btf__type_by_id(obj->btf, btf_id);
1820 if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1821 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1822 btf_kind_str(src_t), sym_name);
1823 return false;
1824 }
1825
1826 /* deal with .maps definitions specially */
1827 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1828 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1829
1830 if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1831 linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1832 return false;
1833
1834 return true;
1835 }
1836
btf_is_non_static(const struct btf_type * t)1837 static bool btf_is_non_static(const struct btf_type *t)
1838 {
1839 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1840 || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1841 }
1842
find_glob_sym_btf(struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int * out_btf_sec_id,int * out_btf_id)1843 static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1844 int *out_btf_sec_id, int *out_btf_id)
1845 {
1846 int i, j, n, m, btf_id = 0;
1847 const struct btf_type *t;
1848 const struct btf_var_secinfo *vi;
1849 const char *name;
1850
1851 if (!obj->btf) {
1852 pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1853 return -EINVAL;
1854 }
1855
1856 n = btf__type_cnt(obj->btf);
1857 for (i = 1; i < n; i++) {
1858 t = btf__type_by_id(obj->btf, i);
1859
1860 /* some global and extern FUNCs and VARs might not be associated with any
1861 * DATASEC, so try to detect them in the same pass
1862 */
1863 if (btf_is_non_static(t)) {
1864 name = btf__str_by_offset(obj->btf, t->name_off);
1865 if (strcmp(name, sym_name) != 0)
1866 continue;
1867
1868 /* remember and still try to find DATASEC */
1869 btf_id = i;
1870 continue;
1871 }
1872
1873 if (!btf_is_datasec(t))
1874 continue;
1875
1876 vi = btf_var_secinfos(t);
1877 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1878 t = btf__type_by_id(obj->btf, vi->type);
1879 name = btf__str_by_offset(obj->btf, t->name_off);
1880
1881 if (strcmp(name, sym_name) != 0)
1882 continue;
1883 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1884 continue;
1885 if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1886 continue;
1887
1888 if (btf_id && btf_id != vi->type) {
1889 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1890 sym_name, btf_id, vi->type);
1891 return -EINVAL;
1892 }
1893
1894 *out_btf_sec_id = i;
1895 *out_btf_id = vi->type;
1896
1897 return 0;
1898 }
1899 }
1900
1901 /* free-floating extern or global FUNC */
1902 if (btf_id) {
1903 *out_btf_sec_id = 0;
1904 *out_btf_id = btf_id;
1905 return 0;
1906 }
1907
1908 pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1909 return -ENOENT;
1910 }
1911
find_src_sec_by_name(struct src_obj * obj,const char * sec_name)1912 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1913 {
1914 struct src_sec *sec;
1915 int i;
1916
1917 for (i = 1; i < obj->sec_cnt; i++) {
1918 sec = &obj->secs[i];
1919
1920 if (strcmp(sec->sec_name, sec_name) == 0)
1921 return sec;
1922 }
1923
1924 return NULL;
1925 }
1926
complete_extern_btf_info(struct btf * dst_btf,int dst_id,struct btf * src_btf,int src_id)1927 static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1928 struct btf *src_btf, int src_id)
1929 {
1930 struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1931 struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1932 struct btf_param *src_p, *dst_p;
1933 const char *s;
1934 int i, n, off;
1935
1936 /* We already made sure that source and destination types (FUNC or
1937 * VAR) match in terms of types and argument names.
1938 */
1939 if (btf_is_var(dst_t)) {
1940 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1941 return 0;
1942 }
1943
1944 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1945
1946 /* now onto FUNC_PROTO types */
1947 src_t = btf_type_by_id(src_btf, src_t->type);
1948 dst_t = btf_type_by_id(dst_btf, dst_t->type);
1949
1950 /* Fill in all the argument names, which for extern FUNCs are missing.
1951 * We'll end up with two copies of FUNCs/VARs for externs, but that
1952 * will be taken care of by BTF dedup at the very end.
1953 * It might be that BTF types for extern in one file has less/more BTF
1954 * information (e.g., FWD instead of full STRUCT/UNION information),
1955 * but that should be (in most cases, subject to BTF dedup rules)
1956 * handled and resolved by BTF dedup algorithm as well, so we won't
1957 * worry about it. Our only job is to make sure that argument names
1958 * are populated on both sides, otherwise BTF dedup will pedantically
1959 * consider them different.
1960 */
1961 src_p = btf_params(src_t);
1962 dst_p = btf_params(dst_t);
1963 for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1964 if (!src_p->name_off)
1965 continue;
1966
1967 /* src_btf has more complete info, so add name to dst_btf */
1968 s = btf__str_by_offset(src_btf, src_p->name_off);
1969 off = btf__add_str(dst_btf, s);
1970 if (off < 0)
1971 return off;
1972 dst_p->name_off = off;
1973 }
1974 return 0;
1975 }
1976
sym_update_bind(Elf64_Sym * sym,int sym_bind)1977 static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1978 {
1979 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1980 }
1981
sym_update_type(Elf64_Sym * sym,int sym_type)1982 static void sym_update_type(Elf64_Sym *sym, int sym_type)
1983 {
1984 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1985 }
1986
sym_update_visibility(Elf64_Sym * sym,int sym_vis)1987 static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1988 {
1989 /* libelf doesn't provide setters for ST_VISIBILITY,
1990 * but it is stored in the lower 2 bits of st_other
1991 */
1992 sym->st_other &= ~0x03;
1993 sym->st_other |= sym_vis;
1994 }
1995
linker_append_elf_sym(struct bpf_linker * linker,struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int src_sym_idx)1996 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1997 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1998 {
1999 struct src_sec *src_sec = NULL;
2000 struct dst_sec *dst_sec = NULL;
2001 struct glob_sym *glob_sym = NULL;
2002 int name_off, sym_type, sym_bind, sym_vis, err;
2003 int btf_sec_id = 0, btf_id = 0;
2004 size_t dst_sym_idx;
2005 Elf64_Sym *dst_sym;
2006 bool sym_is_extern;
2007
2008 sym_type = ELF64_ST_TYPE(sym->st_info);
2009 sym_bind = ELF64_ST_BIND(sym->st_info);
2010 sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
2011 sym_is_extern = sym->st_shndx == SHN_UNDEF;
2012
2013 if (sym_is_extern) {
2014 if (!obj->btf) {
2015 pr_warn("externs without BTF info are not supported\n");
2016 return -ENOTSUP;
2017 }
2018 } else if (sym->st_shndx < SHN_LORESERVE) {
2019 src_sec = &obj->secs[sym->st_shndx];
2020 if (src_sec->skipped)
2021 return 0;
2022 dst_sec = &linker->secs[src_sec->dst_id];
2023
2024 /* allow only one STT_SECTION symbol per section */
2025 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
2026 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
2027 return 0;
2028 }
2029 }
2030
2031 if (sym_bind == STB_LOCAL)
2032 goto add_sym;
2033
2034 /* find matching BTF info */
2035 err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
2036 if (err)
2037 return err;
2038
2039 if (sym_is_extern && btf_sec_id) {
2040 const char *sec_name = NULL;
2041 const struct btf_type *t;
2042
2043 t = btf__type_by_id(obj->btf, btf_sec_id);
2044 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2045
2046 /* Clang puts unannotated extern vars into
2047 * '.extern' BTF DATASEC. Treat them the same
2048 * as unannotated extern funcs (which are
2049 * currently not put into any DATASECs).
2050 * Those don't have associated src_sec/dst_sec.
2051 */
2052 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
2053 src_sec = find_src_sec_by_name(obj, sec_name);
2054 if (!src_sec) {
2055 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
2056 return -ENOENT;
2057 }
2058 dst_sec = &linker->secs[src_sec->dst_id];
2059 }
2060 }
2061
2062 glob_sym = find_glob_sym(linker, sym_name);
2063 if (glob_sym) {
2064 /* Preventively resolve to existing symbol. This is
2065 * needed for further relocation symbol remapping in
2066 * the next step of linking.
2067 */
2068 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2069
2070 /* If both symbols are non-externs, at least one of
2071 * them has to be STB_WEAK, otherwise they are in
2072 * a conflict with each other.
2073 */
2074 if (!sym_is_extern && !glob_sym->is_extern
2075 && !glob_sym->is_weak && sym_bind != STB_WEAK) {
2076 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
2077 src_sym_idx, sym_name, obj->filename);
2078 return -EINVAL;
2079 }
2080
2081 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
2082 return -EINVAL;
2083
2084 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
2085
2086 /* If new symbol is strong, then force dst_sym to be strong as
2087 * well; this way a mix of weak and non-weak extern
2088 * definitions will end up being strong.
2089 */
2090 if (sym_bind == STB_GLOBAL) {
2091 /* We still need to preserve type (NOTYPE or
2092 * OBJECT/FUNC, depending on whether the symbol is
2093 * extern or not)
2094 */
2095 sym_update_bind(dst_sym, STB_GLOBAL);
2096 glob_sym->is_weak = false;
2097 }
2098
2099 /* Non-default visibility is "contaminating", with stricter
2100 * visibility overwriting more permissive ones, even if more
2101 * permissive visibility comes from just an extern definition.
2102 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
2103 * ensured by ELF symbol sanity checks above.
2104 */
2105 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
2106 sym_update_visibility(dst_sym, sym_vis);
2107
2108 /* If the new symbol is extern, then regardless if
2109 * existing symbol is extern or resolved global, just
2110 * keep the existing one untouched.
2111 */
2112 if (sym_is_extern)
2113 return 0;
2114
2115 /* If existing symbol is a strong resolved symbol, bail out,
2116 * because we lost resolution battle have nothing to
2117 * contribute. We already checked above that there is no
2118 * strong-strong conflict. We also already tightened binding
2119 * and visibility, so nothing else to contribute at that point.
2120 */
2121 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
2122 return 0;
2123
2124 /* At this point, new symbol is strong non-extern,
2125 * so overwrite glob_sym with new symbol information.
2126 * Preserve binding and visibility.
2127 */
2128 sym_update_type(dst_sym, sym_type);
2129 dst_sym->st_shndx = dst_sec->sec_idx;
2130 dst_sym->st_value = src_sec->dst_off + sym->st_value;
2131 dst_sym->st_size = sym->st_size;
2132
2133 /* see comment below about dst_sec->id vs dst_sec->sec_idx */
2134 glob_sym->sec_id = dst_sec->id;
2135 glob_sym->is_extern = false;
2136
2137 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
2138 obj->btf, btf_id))
2139 return -EINVAL;
2140
2141 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
2142 glob_sym->underlying_btf_id = 0;
2143
2144 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2145 return 0;
2146 }
2147
2148 add_sym:
2149 name_off = strset__add_str(linker->strtab_strs, sym_name);
2150 if (name_off < 0)
2151 return name_off;
2152
2153 dst_sym = add_new_sym(linker, &dst_sym_idx);
2154 if (!dst_sym)
2155 return -ENOMEM;
2156
2157 dst_sym->st_name = name_off;
2158 dst_sym->st_info = sym->st_info;
2159 dst_sym->st_other = sym->st_other;
2160 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
2161 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
2162 dst_sym->st_size = sym->st_size;
2163
2164 obj->sym_map[src_sym_idx] = dst_sym_idx;
2165
2166 if (sym_type == STT_SECTION && dst_sym) {
2167 dst_sec->sec_sym_idx = dst_sym_idx;
2168 dst_sym->st_value = 0;
2169 }
2170
2171 if (sym_bind != STB_LOCAL) {
2172 glob_sym = add_glob_sym(linker);
2173 if (!glob_sym)
2174 return -ENOMEM;
2175
2176 glob_sym->sym_idx = dst_sym_idx;
2177 /* we use dst_sec->id (and not dst_sec->sec_idx), because
2178 * ephemeral sections (.kconfig, .ksyms, etc) don't have
2179 * sec_idx (as they don't have corresponding ELF section), but
2180 * still have id. .extern doesn't have even ephemeral section
2181 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
2182 */
2183 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
2184 glob_sym->name_off = name_off;
2185 /* we will fill btf_id in during BTF merging step */
2186 glob_sym->btf_id = 0;
2187 glob_sym->is_extern = sym_is_extern;
2188 glob_sym->is_weak = sym_bind == STB_WEAK;
2189 }
2190
2191 return 0;
2192 }
2193
linker_append_elf_relos(struct bpf_linker * linker,struct src_obj * obj)2194 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2195 {
2196 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2197 int i, err;
2198
2199 for (i = 1; i < obj->sec_cnt; i++) {
2200 struct src_sec *src_sec, *src_linked_sec;
2201 struct dst_sec *dst_sec, *dst_linked_sec;
2202 Elf64_Rel *src_rel, *dst_rel;
2203 int j, n;
2204
2205 src_sec = &obj->secs[i];
2206 if (!is_relo_sec(src_sec))
2207 continue;
2208
2209 /* shdr->sh_info points to relocatable section */
2210 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2211 if (src_linked_sec->skipped)
2212 continue;
2213
2214 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2215 if (!dst_sec) {
2216 dst_sec = add_dst_sec(linker, src_sec->sec_name);
2217 if (!dst_sec)
2218 return -ENOMEM;
2219 err = init_sec(linker, dst_sec, src_sec);
2220 if (err) {
2221 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2222 return err;
2223 }
2224 } else if (!secs_match(dst_sec, src_sec)) {
2225 pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2226 return -1;
2227 }
2228
2229 /* shdr->sh_link points to SYMTAB */
2230 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2231
2232 /* shdr->sh_info points to relocated section */
2233 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2234 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2235
2236 src_sec->dst_id = dst_sec->id;
2237 err = extend_sec(linker, dst_sec, src_sec);
2238 if (err)
2239 return err;
2240
2241 src_rel = src_sec->data->d_buf;
2242 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2243 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2244 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2245 size_t src_sym_idx, dst_sym_idx, sym_type;
2246 Elf64_Sym *src_sym;
2247
2248 src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2249 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2250
2251 dst_sym_idx = obj->sym_map[src_sym_idx];
2252 dst_rel->r_offset += src_linked_sec->dst_off;
2253 sym_type = ELF64_R_TYPE(src_rel->r_info);
2254 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2255
2256 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2257 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2258 struct bpf_insn *insn;
2259
2260 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2261 /* calls to the very first static function inside
2262 * .text section at offset 0 will
2263 * reference section symbol, not the
2264 * function symbol. Fix that up,
2265 * otherwise it won't be possible to
2266 * relocate calls to two different
2267 * static functions with the same name
2268 * (rom two different object files)
2269 */
2270 insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2271 if (insn->code == (BPF_JMP | BPF_CALL))
2272 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2273 else
2274 insn->imm += sec->dst_off;
2275 } else {
2276 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2277 return -EINVAL;
2278 }
2279 }
2280
2281 }
2282 }
2283
2284 return 0;
2285 }
2286
find_sym_by_name(struct src_obj * obj,size_t sec_idx,int sym_type,const char * sym_name)2287 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2288 int sym_type, const char *sym_name)
2289 {
2290 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2291 Elf64_Sym *sym = symtab->data->d_buf;
2292 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2293 int str_sec_idx = symtab->shdr->sh_link;
2294 const char *name;
2295
2296 for (i = 0; i < n; i++, sym++) {
2297 if (sym->st_shndx != sec_idx)
2298 continue;
2299 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2300 continue;
2301
2302 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2303 if (!name)
2304 return NULL;
2305
2306 if (strcmp(sym_name, name) != 0)
2307 continue;
2308
2309 return sym;
2310 }
2311
2312 return NULL;
2313 }
2314
linker_fixup_btf(struct src_obj * obj)2315 static int linker_fixup_btf(struct src_obj *obj)
2316 {
2317 const char *sec_name;
2318 struct src_sec *sec;
2319 int i, j, n, m;
2320
2321 if (!obj->btf)
2322 return 0;
2323
2324 n = btf__type_cnt(obj->btf);
2325 for (i = 1; i < n; i++) {
2326 struct btf_var_secinfo *vi;
2327 struct btf_type *t;
2328
2329 t = btf_type_by_id(obj->btf, i);
2330 if (btf_kind(t) != BTF_KIND_DATASEC)
2331 continue;
2332
2333 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2334 sec = find_src_sec_by_name(obj, sec_name);
2335 if (sec) {
2336 /* record actual section size, unless ephemeral */
2337 if (sec->shdr)
2338 t->size = sec->shdr->sh_size;
2339 } else {
2340 /* BTF can have some sections that are not represented
2341 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2342 * for special extern variables.
2343 *
2344 * For all but one such special (ephemeral)
2345 * sections, we pre-create "section shells" to be able
2346 * to keep track of extra per-section metadata later
2347 * (e.g., those BTF extern variables).
2348 *
2349 * .extern is even more special, though, because it
2350 * contains extern variables that need to be resolved
2351 * by static linker, not libbpf and kernel. When such
2352 * externs are resolved, we are going to remove them
2353 * from .extern BTF section and might end up not
2354 * needing it at all. Each resolved extern should have
2355 * matching non-extern VAR/FUNC in other sections.
2356 *
2357 * We do support leaving some of the externs
2358 * unresolved, though, to support cases of building
2359 * libraries, which will later be linked against final
2360 * BPF applications. So if at finalization we still
2361 * see unresolved externs, we'll create .extern
2362 * section on our own.
2363 */
2364 if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2365 continue;
2366
2367 sec = add_src_sec(obj, sec_name);
2368 if (!sec)
2369 return -ENOMEM;
2370
2371 sec->ephemeral = true;
2372 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2373 }
2374
2375 /* remember ELF section and its BTF type ID match */
2376 sec->sec_type_id = i;
2377
2378 /* fix up variable offsets */
2379 vi = btf_var_secinfos(t);
2380 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2381 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2382 const char *var_name;
2383 int var_linkage;
2384 Elf64_Sym *sym;
2385
2386 /* could be a variable or function */
2387 if (!btf_is_var(vt))
2388 continue;
2389
2390 var_name = btf__str_by_offset(obj->btf, vt->name_off);
2391 var_linkage = btf_var(vt)->linkage;
2392
2393 /* no need to patch up static or extern vars */
2394 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2395 continue;
2396
2397 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2398 if (!sym) {
2399 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2400 return -ENOENT;
2401 }
2402
2403 vi->offset = sym->st_value;
2404 }
2405 }
2406
2407 return 0;
2408 }
2409
linker_append_btf(struct bpf_linker * linker,struct src_obj * obj)2410 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2411 {
2412 const struct btf_type *t;
2413 int i, j, n, start_id, id, err;
2414 const char *name;
2415
2416 if (!obj->btf)
2417 return 0;
2418
2419 start_id = btf__type_cnt(linker->btf);
2420 n = btf__type_cnt(obj->btf);
2421
2422 obj->btf_type_map = calloc(n + 1, sizeof(int));
2423 if (!obj->btf_type_map)
2424 return -ENOMEM;
2425
2426 for (i = 1; i < n; i++) {
2427 struct glob_sym *glob_sym = NULL;
2428
2429 t = btf__type_by_id(obj->btf, i);
2430
2431 /* DATASECs are handled specially below */
2432 if (btf_kind(t) == BTF_KIND_DATASEC)
2433 continue;
2434
2435 if (btf_is_non_static(t)) {
2436 /* there should be glob_sym already */
2437 name = btf__str_by_offset(obj->btf, t->name_off);
2438 glob_sym = find_glob_sym(linker, name);
2439
2440 /* VARs without corresponding glob_sym are those that
2441 * belong to skipped/deduplicated sections (i.e.,
2442 * license and version), so just skip them
2443 */
2444 if (!glob_sym)
2445 continue;
2446
2447 /* linker_append_elf_sym() might have requested
2448 * updating underlying type ID, if extern was resolved
2449 * to strong symbol or weak got upgraded to non-weak
2450 */
2451 if (glob_sym->underlying_btf_id == 0)
2452 glob_sym->underlying_btf_id = -t->type;
2453
2454 /* globals from previous object files that match our
2455 * VAR/FUNC already have a corresponding associated
2456 * BTF type, so just make sure to use it
2457 */
2458 if (glob_sym->btf_id) {
2459 /* reuse existing BTF type for global var/func */
2460 obj->btf_type_map[i] = glob_sym->btf_id;
2461 continue;
2462 }
2463 }
2464
2465 id = btf__add_type(linker->btf, obj->btf, t);
2466 if (id < 0) {
2467 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2468 return id;
2469 }
2470
2471 obj->btf_type_map[i] = id;
2472
2473 /* record just appended BTF type for var/func */
2474 if (glob_sym) {
2475 glob_sym->btf_id = id;
2476 glob_sym->underlying_btf_id = -t->type;
2477 }
2478 }
2479
2480 /* remap all the types except DATASECs */
2481 n = btf__type_cnt(linker->btf);
2482 for (i = start_id; i < n; i++) {
2483 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2484 struct btf_field_iter it;
2485 __u32 *type_id;
2486
2487 err = btf_field_iter_init(&it, dst_t, BTF_FIELD_ITER_IDS);
2488 if (err)
2489 return err;
2490
2491 while ((type_id = btf_field_iter_next(&it))) {
2492 int new_id = obj->btf_type_map[*type_id];
2493
2494 /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2495 if (new_id == 0 && *type_id != 0) {
2496 pr_warn("failed to find new ID mapping for original BTF type ID %u\n",
2497 *type_id);
2498 return -EINVAL;
2499 }
2500
2501 *type_id = obj->btf_type_map[*type_id];
2502 }
2503 }
2504
2505 /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2506 * actual type), if necessary
2507 */
2508 for (i = 0; i < linker->glob_sym_cnt; i++) {
2509 struct glob_sym *glob_sym = &linker->glob_syms[i];
2510 struct btf_type *glob_t;
2511
2512 if (glob_sym->underlying_btf_id >= 0)
2513 continue;
2514
2515 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2516
2517 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2518 glob_t->type = glob_sym->underlying_btf_id;
2519 }
2520
2521 /* append DATASEC info */
2522 for (i = 1; i < obj->sec_cnt; i++) {
2523 struct src_sec *src_sec;
2524 struct dst_sec *dst_sec;
2525 const struct btf_var_secinfo *src_var;
2526 struct btf_var_secinfo *dst_var;
2527
2528 src_sec = &obj->secs[i];
2529 if (!src_sec->sec_type_id || src_sec->skipped)
2530 continue;
2531 dst_sec = &linker->secs[src_sec->dst_id];
2532
2533 /* Mark section as having BTF regardless of the presence of
2534 * variables. In some cases compiler might generate empty BTF
2535 * with no variables information. E.g., when promoting local
2536 * array/structure variable initial values and BPF object
2537 * file otherwise has no read-only static variables in
2538 * .rodata. We need to preserve such empty BTF and just set
2539 * correct section size.
2540 */
2541 dst_sec->has_btf = true;
2542
2543 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2544 src_var = btf_var_secinfos(t);
2545 n = btf_vlen(t);
2546 for (j = 0; j < n; j++, src_var++) {
2547 void *sec_vars = dst_sec->sec_vars;
2548 int new_id = obj->btf_type_map[src_var->type];
2549 struct glob_sym *glob_sym = NULL;
2550
2551 t = btf_type_by_id(linker->btf, new_id);
2552 if (btf_is_non_static(t)) {
2553 name = btf__str_by_offset(linker->btf, t->name_off);
2554 glob_sym = find_glob_sym(linker, name);
2555 if (glob_sym->sec_id != dst_sec->id) {
2556 pr_warn("global '%s': section mismatch %d vs %d\n",
2557 name, glob_sym->sec_id, dst_sec->id);
2558 return -EINVAL;
2559 }
2560 }
2561
2562 /* If there is already a member (VAR or FUNC) mapped
2563 * to the same type, don't add a duplicate entry.
2564 * This will happen when multiple object files define
2565 * the same extern VARs/FUNCs.
2566 */
2567 if (glob_sym && glob_sym->var_idx >= 0) {
2568 __s64 sz;
2569
2570 /* FUNCs don't have size, nothing to update */
2571 if (btf_is_func(t))
2572 continue;
2573
2574 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2575 /* Because underlying BTF type might have
2576 * changed, so might its size have changed, so
2577 * re-calculate and update it in sec_var.
2578 */
2579 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2580 if (sz < 0) {
2581 pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2582 name, (int)sz);
2583 return -EINVAL;
2584 }
2585 dst_var->size = sz;
2586 continue;
2587 }
2588
2589 sec_vars = libbpf_reallocarray(sec_vars,
2590 dst_sec->sec_var_cnt + 1,
2591 sizeof(*dst_sec->sec_vars));
2592 if (!sec_vars)
2593 return -ENOMEM;
2594
2595 dst_sec->sec_vars = sec_vars;
2596 dst_sec->sec_var_cnt++;
2597
2598 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2599 dst_var->type = obj->btf_type_map[src_var->type];
2600 dst_var->size = src_var->size;
2601 dst_var->offset = src_sec->dst_off + src_var->offset;
2602
2603 if (glob_sym)
2604 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2605 }
2606 }
2607
2608 return 0;
2609 }
2610
add_btf_ext_rec(struct btf_ext_sec_data * ext_data,const void * src_rec)2611 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2612 {
2613 void *tmp;
2614
2615 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2616 if (!tmp)
2617 return NULL;
2618 ext_data->recs = tmp;
2619
2620 tmp += ext_data->rec_cnt * ext_data->rec_sz;
2621 memcpy(tmp, src_rec, ext_data->rec_sz);
2622
2623 ext_data->rec_cnt++;
2624
2625 return tmp;
2626 }
2627
linker_append_btf_ext(struct bpf_linker * linker,struct src_obj * obj)2628 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2629 {
2630 const struct btf_ext_info_sec *ext_sec;
2631 const char *sec_name, *s;
2632 struct src_sec *src_sec;
2633 struct dst_sec *dst_sec;
2634 int rec_sz, str_off, i;
2635
2636 if (!obj->btf_ext)
2637 return 0;
2638
2639 rec_sz = obj->btf_ext->func_info.rec_size;
2640 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2641 struct bpf_func_info_min *src_rec, *dst_rec;
2642
2643 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2644 src_sec = find_src_sec_by_name(obj, sec_name);
2645 if (!src_sec) {
2646 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2647 return -EINVAL;
2648 }
2649 dst_sec = &linker->secs[src_sec->dst_id];
2650
2651 if (dst_sec->func_info.rec_sz == 0)
2652 dst_sec->func_info.rec_sz = rec_sz;
2653 if (dst_sec->func_info.rec_sz != rec_sz) {
2654 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2655 return -EINVAL;
2656 }
2657
2658 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2659 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2660 if (!dst_rec)
2661 return -ENOMEM;
2662
2663 dst_rec->insn_off += src_sec->dst_off;
2664 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2665 }
2666 }
2667
2668 rec_sz = obj->btf_ext->line_info.rec_size;
2669 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2670 struct bpf_line_info_min *src_rec, *dst_rec;
2671
2672 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2673 src_sec = find_src_sec_by_name(obj, sec_name);
2674 if (!src_sec) {
2675 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2676 return -EINVAL;
2677 }
2678 dst_sec = &linker->secs[src_sec->dst_id];
2679
2680 if (dst_sec->line_info.rec_sz == 0)
2681 dst_sec->line_info.rec_sz = rec_sz;
2682 if (dst_sec->line_info.rec_sz != rec_sz) {
2683 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2684 return -EINVAL;
2685 }
2686
2687 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2688 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2689 if (!dst_rec)
2690 return -ENOMEM;
2691
2692 dst_rec->insn_off += src_sec->dst_off;
2693
2694 s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2695 str_off = btf__add_str(linker->btf, s);
2696 if (str_off < 0)
2697 return -ENOMEM;
2698 dst_rec->file_name_off = str_off;
2699
2700 s = btf__str_by_offset(obj->btf, src_rec->line_off);
2701 str_off = btf__add_str(linker->btf, s);
2702 if (str_off < 0)
2703 return -ENOMEM;
2704 dst_rec->line_off = str_off;
2705
2706 /* dst_rec->line_col is fine */
2707 }
2708 }
2709
2710 rec_sz = obj->btf_ext->core_relo_info.rec_size;
2711 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2712 struct bpf_core_relo *src_rec, *dst_rec;
2713
2714 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2715 src_sec = find_src_sec_by_name(obj, sec_name);
2716 if (!src_sec) {
2717 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2718 return -EINVAL;
2719 }
2720 dst_sec = &linker->secs[src_sec->dst_id];
2721
2722 if (dst_sec->core_relo_info.rec_sz == 0)
2723 dst_sec->core_relo_info.rec_sz = rec_sz;
2724 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2725 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2726 return -EINVAL;
2727 }
2728
2729 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2730 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2731 if (!dst_rec)
2732 return -ENOMEM;
2733
2734 dst_rec->insn_off += src_sec->dst_off;
2735 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2736
2737 s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2738 str_off = btf__add_str(linker->btf, s);
2739 if (str_off < 0)
2740 return -ENOMEM;
2741 dst_rec->access_str_off = str_off;
2742
2743 /* dst_rec->kind is fine */
2744 }
2745 }
2746
2747 return 0;
2748 }
2749
bpf_linker__finalize(struct bpf_linker * linker)2750 int bpf_linker__finalize(struct bpf_linker *linker)
2751 {
2752 struct dst_sec *sec;
2753 size_t strs_sz;
2754 const void *strs;
2755 int err, i;
2756
2757 if (!linker->elf)
2758 return libbpf_err(-EINVAL);
2759
2760 err = finalize_btf(linker);
2761 if (err)
2762 return libbpf_err(err);
2763
2764 /* Finalize strings */
2765 strs_sz = strset__data_size(linker->strtab_strs);
2766 strs = strset__data(linker->strtab_strs);
2767
2768 sec = &linker->secs[linker->strtab_sec_idx];
2769 sec->data->d_align = 1;
2770 sec->data->d_off = 0LL;
2771 sec->data->d_buf = (void *)strs;
2772 sec->data->d_type = ELF_T_BYTE;
2773 sec->data->d_size = strs_sz;
2774 sec->shdr->sh_size = strs_sz;
2775
2776 for (i = 1; i < linker->sec_cnt; i++) {
2777 sec = &linker->secs[i];
2778
2779 /* STRTAB is handled specially above */
2780 if (sec->sec_idx == linker->strtab_sec_idx)
2781 continue;
2782
2783 /* special ephemeral sections (.ksyms, .kconfig, etc) */
2784 if (!sec->scn)
2785 continue;
2786
2787 /* restore sections with bpf insns to target byte-order */
2788 if (linker->swapped_endian && is_exec_sec(sec))
2789 exec_sec_bswap(sec->raw_data, sec->sec_sz);
2790
2791 sec->data->d_buf = sec->raw_data;
2792 }
2793
2794 /* Finalize ELF layout */
2795 if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2796 err = -EINVAL;
2797 pr_warn_elf("failed to finalize ELF layout");
2798 return libbpf_err(err);
2799 }
2800
2801 /* Write out final ELF contents */
2802 if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2803 err = -EINVAL;
2804 pr_warn_elf("failed to write ELF contents");
2805 return libbpf_err(err);
2806 }
2807
2808 elf_end(linker->elf);
2809 linker->elf = NULL;
2810
2811 if (linker->fd_is_owned)
2812 close(linker->fd);
2813 linker->fd = -1;
2814
2815 return 0;
2816 }
2817
emit_elf_data_sec(struct bpf_linker * linker,const char * sec_name,size_t align,const void * raw_data,size_t raw_sz)2818 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2819 size_t align, const void *raw_data, size_t raw_sz)
2820 {
2821 Elf_Scn *scn;
2822 Elf_Data *data;
2823 Elf64_Shdr *shdr;
2824 int name_off;
2825
2826 name_off = strset__add_str(linker->strtab_strs, sec_name);
2827 if (name_off < 0)
2828 return name_off;
2829
2830 scn = elf_newscn(linker->elf);
2831 if (!scn)
2832 return -ENOMEM;
2833 data = elf_newdata(scn);
2834 if (!data)
2835 return -ENOMEM;
2836 shdr = elf64_getshdr(scn);
2837 if (!shdr)
2838 return -EINVAL;
2839
2840 shdr->sh_name = name_off;
2841 shdr->sh_type = SHT_PROGBITS;
2842 shdr->sh_flags = 0;
2843 shdr->sh_size = raw_sz;
2844 shdr->sh_link = 0;
2845 shdr->sh_info = 0;
2846 shdr->sh_addralign = align;
2847 shdr->sh_entsize = 0;
2848
2849 data->d_type = ELF_T_BYTE;
2850 data->d_size = raw_sz;
2851 data->d_buf = (void *)raw_data;
2852 data->d_align = align;
2853 data->d_off = 0;
2854
2855 return 0;
2856 }
2857
finalize_btf(struct bpf_linker * linker)2858 static int finalize_btf(struct bpf_linker *linker)
2859 {
2860 enum btf_endianness link_endianness;
2861 LIBBPF_OPTS(btf_dedup_opts, opts);
2862 struct btf *btf = linker->btf;
2863 const void *raw_data;
2864 int i, j, id, err;
2865 __u32 raw_sz;
2866
2867 /* bail out if no BTF data was produced */
2868 if (btf__type_cnt(linker->btf) == 1)
2869 return 0;
2870
2871 for (i = 1; i < linker->sec_cnt; i++) {
2872 struct dst_sec *sec = &linker->secs[i];
2873
2874 if (!sec->has_btf)
2875 continue;
2876
2877 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2878 if (id < 0) {
2879 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2880 sec->sec_name, id);
2881 return id;
2882 }
2883
2884 for (j = 0; j < sec->sec_var_cnt; j++) {
2885 struct btf_var_secinfo *vi = &sec->sec_vars[j];
2886
2887 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2888 return -EINVAL;
2889 }
2890 }
2891
2892 err = finalize_btf_ext(linker);
2893 if (err) {
2894 pr_warn(".BTF.ext generation failed: %s\n", errstr(err));
2895 return err;
2896 }
2897
2898 opts.btf_ext = linker->btf_ext;
2899 err = btf__dedup(linker->btf, &opts);
2900 if (err) {
2901 pr_warn("BTF dedup failed: %s\n", errstr(err));
2902 return err;
2903 }
2904
2905 /* Set .BTF and .BTF.ext output byte order */
2906 link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ?
2907 BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
2908 btf__set_endianness(linker->btf, link_endianness);
2909 if (linker->btf_ext)
2910 btf_ext__set_endianness(linker->btf_ext, link_endianness);
2911
2912 /* Emit .BTF section */
2913 raw_data = btf__raw_data(linker->btf, &raw_sz);
2914 if (!raw_data)
2915 return -ENOMEM;
2916
2917 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2918 if (err) {
2919 pr_warn("failed to write out .BTF ELF section: %s\n", errstr(err));
2920 return err;
2921 }
2922
2923 /* Emit .BTF.ext section */
2924 if (linker->btf_ext) {
2925 raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);
2926 if (!raw_data)
2927 return -ENOMEM;
2928
2929 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2930 if (err) {
2931 pr_warn("failed to write out .BTF.ext ELF section: %s\n", errstr(err));
2932 return err;
2933 }
2934 }
2935
2936 return 0;
2937 }
2938
emit_btf_ext_data(struct bpf_linker * linker,void * output,const char * sec_name,struct btf_ext_sec_data * sec_data)2939 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2940 const char *sec_name, struct btf_ext_sec_data *sec_data)
2941 {
2942 struct btf_ext_info_sec *sec_info;
2943 void *cur = output;
2944 int str_off;
2945 size_t sz;
2946
2947 if (!sec_data->rec_cnt)
2948 return 0;
2949
2950 str_off = btf__add_str(linker->btf, sec_name);
2951 if (str_off < 0)
2952 return -ENOMEM;
2953
2954 sec_info = cur;
2955 sec_info->sec_name_off = str_off;
2956 sec_info->num_info = sec_data->rec_cnt;
2957 cur += sizeof(struct btf_ext_info_sec);
2958
2959 sz = sec_data->rec_cnt * sec_data->rec_sz;
2960 memcpy(cur, sec_data->recs, sz);
2961 cur += sz;
2962
2963 return cur - output;
2964 }
2965
finalize_btf_ext(struct bpf_linker * linker)2966 static int finalize_btf_ext(struct bpf_linker *linker)
2967 {
2968 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2969 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2970 struct btf_ext_header *hdr;
2971 void *data, *cur;
2972 int i, err, sz;
2973
2974 /* validate that all sections have the same .BTF.ext record sizes
2975 * and calculate total data size for each type of data (func info,
2976 * line info, core relos)
2977 */
2978 for (i = 1; i < linker->sec_cnt; i++) {
2979 struct dst_sec *sec = &linker->secs[i];
2980
2981 if (sec->func_info.rec_cnt) {
2982 if (func_rec_sz == 0)
2983 func_rec_sz = sec->func_info.rec_sz;
2984 if (func_rec_sz != sec->func_info.rec_sz) {
2985 pr_warn("mismatch in func_info record size %zu != %u\n",
2986 func_rec_sz, sec->func_info.rec_sz);
2987 return -EINVAL;
2988 }
2989
2990 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2991 }
2992 if (sec->line_info.rec_cnt) {
2993 if (line_rec_sz == 0)
2994 line_rec_sz = sec->line_info.rec_sz;
2995 if (line_rec_sz != sec->line_info.rec_sz) {
2996 pr_warn("mismatch in line_info record size %zu != %u\n",
2997 line_rec_sz, sec->line_info.rec_sz);
2998 return -EINVAL;
2999 }
3000
3001 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
3002 }
3003 if (sec->core_relo_info.rec_cnt) {
3004 if (core_relo_rec_sz == 0)
3005 core_relo_rec_sz = sec->core_relo_info.rec_sz;
3006 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
3007 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
3008 core_relo_rec_sz, sec->core_relo_info.rec_sz);
3009 return -EINVAL;
3010 }
3011
3012 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
3013 }
3014 }
3015
3016 if (!funcs_sz && !lines_sz && !core_relos_sz)
3017 return 0;
3018
3019 total_sz += sizeof(struct btf_ext_header);
3020 if (funcs_sz) {
3021 funcs_sz += sizeof(__u32); /* record size prefix */
3022 total_sz += funcs_sz;
3023 }
3024 if (lines_sz) {
3025 lines_sz += sizeof(__u32); /* record size prefix */
3026 total_sz += lines_sz;
3027 }
3028 if (core_relos_sz) {
3029 core_relos_sz += sizeof(__u32); /* record size prefix */
3030 total_sz += core_relos_sz;
3031 }
3032
3033 cur = data = calloc(1, total_sz);
3034 if (!data)
3035 return -ENOMEM;
3036
3037 hdr = cur;
3038 hdr->magic = BTF_MAGIC;
3039 hdr->version = BTF_VERSION;
3040 hdr->flags = 0;
3041 hdr->hdr_len = sizeof(struct btf_ext_header);
3042 cur += sizeof(struct btf_ext_header);
3043
3044 /* All offsets are in bytes relative to the end of this header */
3045 hdr->func_info_off = 0;
3046 hdr->func_info_len = funcs_sz;
3047 hdr->line_info_off = funcs_sz;
3048 hdr->line_info_len = lines_sz;
3049 hdr->core_relo_off = funcs_sz + lines_sz;
3050 hdr->core_relo_len = core_relos_sz;
3051
3052 if (funcs_sz) {
3053 *(__u32 *)cur = func_rec_sz;
3054 cur += sizeof(__u32);
3055
3056 for (i = 1; i < linker->sec_cnt; i++) {
3057 struct dst_sec *sec = &linker->secs[i];
3058
3059 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
3060 if (sz < 0) {
3061 err = sz;
3062 goto out;
3063 }
3064
3065 cur += sz;
3066 }
3067 }
3068
3069 if (lines_sz) {
3070 *(__u32 *)cur = line_rec_sz;
3071 cur += sizeof(__u32);
3072
3073 for (i = 1; i < linker->sec_cnt; i++) {
3074 struct dst_sec *sec = &linker->secs[i];
3075
3076 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
3077 if (sz < 0) {
3078 err = sz;
3079 goto out;
3080 }
3081
3082 cur += sz;
3083 }
3084 }
3085
3086 if (core_relos_sz) {
3087 *(__u32 *)cur = core_relo_rec_sz;
3088 cur += sizeof(__u32);
3089
3090 for (i = 1; i < linker->sec_cnt; i++) {
3091 struct dst_sec *sec = &linker->secs[i];
3092
3093 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
3094 if (sz < 0) {
3095 err = sz;
3096 goto out;
3097 }
3098
3099 cur += sz;
3100 }
3101 }
3102
3103 linker->btf_ext = btf_ext__new(data, total_sz);
3104 err = libbpf_get_error(linker->btf_ext);
3105 if (err) {
3106 linker->btf_ext = NULL;
3107 pr_warn("failed to parse final .BTF.ext data: %s\n", errstr(err));
3108 goto out;
3109 }
3110
3111 out:
3112 free(data);
3113 return err;
3114 }
3115