1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4 */
5
6 #include <string.h>
7 #include <stdlib.h>
8 #include <inttypes.h>
9 #include <sys/mman.h>
10
11 #include <objtool/builtin.h>
12 #include <objtool/cfi.h>
13 #include <objtool/arch.h>
14 #include <objtool/check.h>
15 #include <objtool/special.h>
16 #include <objtool/warn.h>
17 #include <objtool/endianness.h>
18
19 #include <linux/objtool_types.h>
20 #include <linux/hashtable.h>
21 #include <linux/kernel.h>
22 #include <linux/static_call_types.h>
23 #include <linux/string.h>
24
25 struct alternative {
26 struct alternative *next;
27 struct instruction *insn;
28 bool skip_orig;
29 };
30
31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32
33 static struct cfi_init_state initial_func_cfi;
34 static struct cfi_state init_cfi;
35 static struct cfi_state func_cfi;
36 static struct cfi_state force_undefined_cfi;
37
find_insn(struct objtool_file * file,struct section * sec,unsigned long offset)38 struct instruction *find_insn(struct objtool_file *file,
39 struct section *sec, unsigned long offset)
40 {
41 struct instruction *insn;
42
43 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
44 if (insn->sec == sec && insn->offset == offset)
45 return insn;
46 }
47
48 return NULL;
49 }
50
next_insn_same_sec(struct objtool_file * file,struct instruction * insn)51 struct instruction *next_insn_same_sec(struct objtool_file *file,
52 struct instruction *insn)
53 {
54 if (insn->idx == INSN_CHUNK_MAX)
55 return find_insn(file, insn->sec, insn->offset + insn->len);
56
57 insn++;
58 if (!insn->len)
59 return NULL;
60
61 return insn;
62 }
63
next_insn_same_func(struct objtool_file * file,struct instruction * insn)64 static struct instruction *next_insn_same_func(struct objtool_file *file,
65 struct instruction *insn)
66 {
67 struct instruction *next = next_insn_same_sec(file, insn);
68 struct symbol *func = insn_func(insn);
69
70 if (!func)
71 return NULL;
72
73 if (next && insn_func(next) == func)
74 return next;
75
76 /* Check if we're already in the subfunction: */
77 if (func == func->cfunc)
78 return NULL;
79
80 /* Move to the subfunction: */
81 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
82 }
83
prev_insn_same_sec(struct objtool_file * file,struct instruction * insn)84 static struct instruction *prev_insn_same_sec(struct objtool_file *file,
85 struct instruction *insn)
86 {
87 if (insn->idx == 0) {
88 if (insn->prev_len)
89 return find_insn(file, insn->sec, insn->offset - insn->prev_len);
90 return NULL;
91 }
92
93 return insn - 1;
94 }
95
prev_insn_same_sym(struct objtool_file * file,struct instruction * insn)96 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
97 struct instruction *insn)
98 {
99 struct instruction *prev = prev_insn_same_sec(file, insn);
100
101 if (prev && insn_func(prev) == insn_func(insn))
102 return prev;
103
104 return NULL;
105 }
106
107 #define for_each_insn(file, insn) \
108 for (struct section *__sec, *__fake = (struct section *)1; \
109 __fake; __fake = NULL) \
110 for_each_sec(file, __sec) \
111 sec_for_each_insn(file, __sec, insn)
112
113 #define func_for_each_insn(file, func, insn) \
114 for (insn = find_insn(file, func->sec, func->offset); \
115 insn; \
116 insn = next_insn_same_func(file, insn))
117
118 #define sym_for_each_insn(file, sym, insn) \
119 for (insn = find_insn(file, sym->sec, sym->offset); \
120 insn && insn->offset < sym->offset + sym->len; \
121 insn = next_insn_same_sec(file, insn))
122
123 #define sym_for_each_insn_continue_reverse(file, sym, insn) \
124 for (insn = prev_insn_same_sec(file, insn); \
125 insn && insn->offset >= sym->offset; \
126 insn = prev_insn_same_sec(file, insn))
127
128 #define sec_for_each_insn_from(file, insn) \
129 for (; insn; insn = next_insn_same_sec(file, insn))
130
131 #define sec_for_each_insn_continue(file, insn) \
132 for (insn = next_insn_same_sec(file, insn); insn; \
133 insn = next_insn_same_sec(file, insn))
134
insn_call_dest(struct instruction * insn)135 static inline struct symbol *insn_call_dest(struct instruction *insn)
136 {
137 if (insn->type == INSN_JUMP_DYNAMIC ||
138 insn->type == INSN_CALL_DYNAMIC)
139 return NULL;
140
141 return insn->_call_dest;
142 }
143
insn_jump_table(struct instruction * insn)144 static inline struct reloc *insn_jump_table(struct instruction *insn)
145 {
146 if (insn->type == INSN_JUMP_DYNAMIC ||
147 insn->type == INSN_CALL_DYNAMIC)
148 return insn->_jump_table;
149
150 return NULL;
151 }
152
insn_jump_table_size(struct instruction * insn)153 static inline unsigned long insn_jump_table_size(struct instruction *insn)
154 {
155 if (insn->type == INSN_JUMP_DYNAMIC ||
156 insn->type == INSN_CALL_DYNAMIC)
157 return insn->_jump_table_size;
158
159 return 0;
160 }
161
is_jump_table_jump(struct instruction * insn)162 static bool is_jump_table_jump(struct instruction *insn)
163 {
164 struct alt_group *alt_group = insn->alt_group;
165
166 if (insn_jump_table(insn))
167 return true;
168
169 /* Retpoline alternative for a jump table? */
170 return alt_group && alt_group->orig_group &&
171 insn_jump_table(alt_group->orig_group->first_insn);
172 }
173
is_sibling_call(struct instruction * insn)174 static bool is_sibling_call(struct instruction *insn)
175 {
176 /*
177 * Assume only STT_FUNC calls have jump-tables.
178 */
179 if (insn_func(insn)) {
180 /* An indirect jump is either a sibling call or a jump to a table. */
181 if (insn->type == INSN_JUMP_DYNAMIC)
182 return !is_jump_table_jump(insn);
183 }
184
185 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
186 return (is_static_jump(insn) && insn_call_dest(insn));
187 }
188
189 /*
190 * Checks if a string ends with another.
191 */
str_ends_with(const char * s,const char * sub)192 static bool str_ends_with(const char *s, const char *sub)
193 {
194 const int slen = strlen(s);
195 const int sublen = strlen(sub);
196
197 if (sublen > slen)
198 return 0;
199
200 return !memcmp(s + slen - sublen, sub, sublen);
201 }
202
203 /*
204 * Checks if a function is a Rust "noreturn" one.
205 */
is_rust_noreturn(const struct symbol * func)206 static bool is_rust_noreturn(const struct symbol *func)
207 {
208 /*
209 * If it does not start with "_R", then it is not a Rust symbol.
210 */
211 if (strncmp(func->name, "_R", 2))
212 return false;
213
214 /*
215 * These are just heuristics -- we do not control the precise symbol
216 * name, due to the crate disambiguators (which depend on the compiler)
217 * as well as changes to the source code itself between versions (since
218 * these come from the Rust standard library).
219 */
220 return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") ||
221 str_ends_with(func->name, "_4core6option13unwrap_failed") ||
222 str_ends_with(func->name, "_4core6result13unwrap_failed") ||
223 str_ends_with(func->name, "_4core9panicking5panic") ||
224 str_ends_with(func->name, "_4core9panicking9panic_fmt") ||
225 str_ends_with(func->name, "_4core9panicking14panic_explicit") ||
226 str_ends_with(func->name, "_4core9panicking14panic_nounwind") ||
227 str_ends_with(func->name, "_4core9panicking18panic_bounds_check") ||
228 str_ends_with(func->name, "_4core9panicking19assert_failed_inner") ||
229 str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") ||
230 strstr(func->name, "_4core9panicking13assert_failed") ||
231 strstr(func->name, "_4core9panicking11panic_const24panic_const_") ||
232 (strstr(func->name, "_4core5slice5index24slice_") &&
233 str_ends_with(func->name, "_fail"));
234 }
235
236 /*
237 * This checks to see if the given function is a "noreturn" function.
238 *
239 * For global functions which are outside the scope of this object file, we
240 * have to keep a manual list of them.
241 *
242 * For local functions, we have to detect them manually by simply looking for
243 * the lack of a return instruction.
244 */
__dead_end_function(struct objtool_file * file,struct symbol * func,int recursion)245 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
246 int recursion)
247 {
248 int i;
249 struct instruction *insn;
250 bool empty = true;
251
252 #define NORETURN(func) __stringify(func),
253 static const char * const global_noreturns[] = {
254 #include "noreturns.h"
255 };
256 #undef NORETURN
257
258 if (!func)
259 return false;
260
261 if (func->bind == STB_GLOBAL || func->bind == STB_WEAK) {
262 if (is_rust_noreturn(func))
263 return true;
264
265 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
266 if (!strcmp(func->name, global_noreturns[i]))
267 return true;
268 }
269
270 if (func->bind == STB_WEAK)
271 return false;
272
273 if (!func->len)
274 return false;
275
276 insn = find_insn(file, func->sec, func->offset);
277 if (!insn || !insn_func(insn))
278 return false;
279
280 func_for_each_insn(file, func, insn) {
281 empty = false;
282
283 if (insn->type == INSN_RETURN)
284 return false;
285 }
286
287 if (empty)
288 return false;
289
290 /*
291 * A function can have a sibling call instead of a return. In that
292 * case, the function's dead-end status depends on whether the target
293 * of the sibling call returns.
294 */
295 func_for_each_insn(file, func, insn) {
296 if (is_sibling_call(insn)) {
297 struct instruction *dest = insn->jump_dest;
298
299 if (!dest)
300 /* sibling call to another file */
301 return false;
302
303 /* local sibling call */
304 if (recursion == 5) {
305 /*
306 * Infinite recursion: two functions have
307 * sibling calls to each other. This is a very
308 * rare case. It means they aren't dead ends.
309 */
310 return false;
311 }
312
313 return __dead_end_function(file, insn_func(dest), recursion+1);
314 }
315 }
316
317 return true;
318 }
319
dead_end_function(struct objtool_file * file,struct symbol * func)320 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
321 {
322 return __dead_end_function(file, func, 0);
323 }
324
init_cfi_state(struct cfi_state * cfi)325 static void init_cfi_state(struct cfi_state *cfi)
326 {
327 int i;
328
329 for (i = 0; i < CFI_NUM_REGS; i++) {
330 cfi->regs[i].base = CFI_UNDEFINED;
331 cfi->vals[i].base = CFI_UNDEFINED;
332 }
333 cfi->cfa.base = CFI_UNDEFINED;
334 cfi->drap_reg = CFI_UNDEFINED;
335 cfi->drap_offset = -1;
336 }
337
init_insn_state(struct objtool_file * file,struct insn_state * state,struct section * sec)338 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
339 struct section *sec)
340 {
341 memset(state, 0, sizeof(*state));
342 init_cfi_state(&state->cfi);
343
344 /*
345 * We need the full vmlinux for noinstr validation, otherwise we can
346 * not correctly determine insn_call_dest(insn)->sec (external symbols
347 * do not have a section).
348 */
349 if (opts.link && opts.noinstr && sec)
350 state->noinstr = sec->noinstr;
351 }
352
cfi_alloc(void)353 static struct cfi_state *cfi_alloc(void)
354 {
355 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
356 if (!cfi) {
357 WARN("calloc failed");
358 exit(1);
359 }
360 nr_cfi++;
361 return cfi;
362 }
363
364 static int cfi_bits;
365 static struct hlist_head *cfi_hash;
366
cficmp(struct cfi_state * cfi1,struct cfi_state * cfi2)367 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
368 {
369 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
370 (void *)cfi2 + sizeof(cfi2->hash),
371 sizeof(struct cfi_state) - sizeof(struct hlist_node));
372 }
373
cfi_key(struct cfi_state * cfi)374 static inline u32 cfi_key(struct cfi_state *cfi)
375 {
376 return jhash((void *)cfi + sizeof(cfi->hash),
377 sizeof(*cfi) - sizeof(cfi->hash), 0);
378 }
379
cfi_hash_find_or_add(struct cfi_state * cfi)380 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
381 {
382 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
383 struct cfi_state *obj;
384
385 hlist_for_each_entry(obj, head, hash) {
386 if (!cficmp(cfi, obj)) {
387 nr_cfi_cache++;
388 return obj;
389 }
390 }
391
392 obj = cfi_alloc();
393 *obj = *cfi;
394 hlist_add_head(&obj->hash, head);
395
396 return obj;
397 }
398
cfi_hash_add(struct cfi_state * cfi)399 static void cfi_hash_add(struct cfi_state *cfi)
400 {
401 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
402
403 hlist_add_head(&cfi->hash, head);
404 }
405
cfi_hash_alloc(unsigned long size)406 static void *cfi_hash_alloc(unsigned long size)
407 {
408 cfi_bits = max(10, ilog2(size));
409 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
410 PROT_READ|PROT_WRITE,
411 MAP_PRIVATE|MAP_ANON, -1, 0);
412 if (cfi_hash == (void *)-1L) {
413 WARN("mmap fail cfi_hash");
414 cfi_hash = NULL;
415 } else if (opts.stats) {
416 printf("cfi_bits: %d\n", cfi_bits);
417 }
418
419 return cfi_hash;
420 }
421
422 static unsigned long nr_insns;
423 static unsigned long nr_insns_visited;
424
425 /*
426 * Call the arch-specific instruction decoder for all the instructions and add
427 * them to the global instruction list.
428 */
decode_instructions(struct objtool_file * file)429 static int decode_instructions(struct objtool_file *file)
430 {
431 struct section *sec;
432 struct symbol *func;
433 unsigned long offset;
434 struct instruction *insn;
435 int ret;
436
437 for_each_sec(file, sec) {
438 struct instruction *insns = NULL;
439 u8 prev_len = 0;
440 u8 idx = 0;
441
442 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
443 continue;
444
445 if (strcmp(sec->name, ".altinstr_replacement") &&
446 strcmp(sec->name, ".altinstr_aux") &&
447 strncmp(sec->name, ".discard.", 9))
448 sec->text = true;
449
450 if (!strcmp(sec->name, ".noinstr.text") ||
451 !strcmp(sec->name, ".entry.text") ||
452 !strcmp(sec->name, ".cpuidle.text") ||
453 !strncmp(sec->name, ".text..__x86.", 13))
454 sec->noinstr = true;
455
456 /*
457 * .init.text code is ran before userspace and thus doesn't
458 * strictly need retpolines, except for modules which are
459 * loaded late, they very much do need retpoline in their
460 * .init.text
461 */
462 if (!strcmp(sec->name, ".init.text") && !opts.module)
463 sec->init = true;
464
465 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
466 if (!insns || idx == INSN_CHUNK_MAX) {
467 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
468 if (!insns) {
469 WARN("malloc failed");
470 return -1;
471 }
472 idx = 0;
473 } else {
474 idx++;
475 }
476 insn = &insns[idx];
477 insn->idx = idx;
478
479 INIT_LIST_HEAD(&insn->call_node);
480 insn->sec = sec;
481 insn->offset = offset;
482 insn->prev_len = prev_len;
483
484 ret = arch_decode_instruction(file, sec, offset,
485 sec->sh.sh_size - offset,
486 insn);
487 if (ret)
488 return ret;
489
490 prev_len = insn->len;
491
492 /*
493 * By default, "ud2" is a dead end unless otherwise
494 * annotated, because GCC 7 inserts it for certain
495 * divide-by-zero cases.
496 */
497 if (insn->type == INSN_BUG)
498 insn->dead_end = true;
499
500 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
501 nr_insns++;
502 }
503
504 // printf("%s: last chunk used: %d\n", sec->name, (int)idx);
505
506 sec_for_each_sym(sec, func) {
507 if (func->type != STT_NOTYPE && func->type != STT_FUNC)
508 continue;
509
510 if (func->offset == sec->sh.sh_size) {
511 /* Heuristic: likely an "end" symbol */
512 if (func->type == STT_NOTYPE)
513 continue;
514 WARN("%s(): STT_FUNC at end of section",
515 func->name);
516 return -1;
517 }
518
519 if (func->embedded_insn || func->alias != func)
520 continue;
521
522 if (!find_insn(file, sec, func->offset)) {
523 WARN("%s(): can't find starting instruction",
524 func->name);
525 return -1;
526 }
527
528 sym_for_each_insn(file, func, insn) {
529 insn->sym = func;
530 if (func->type == STT_FUNC &&
531 insn->type == INSN_ENDBR &&
532 list_empty(&insn->call_node)) {
533 if (insn->offset == func->offset) {
534 list_add_tail(&insn->call_node, &file->endbr_list);
535 file->nr_endbr++;
536 } else {
537 file->nr_endbr_int++;
538 }
539 }
540 }
541 }
542 }
543
544 if (opts.stats)
545 printf("nr_insns: %lu\n", nr_insns);
546
547 return 0;
548 }
549
550 /*
551 * Read the pv_ops[] .data table to find the static initialized values.
552 */
add_pv_ops(struct objtool_file * file,const char * symname)553 static int add_pv_ops(struct objtool_file *file, const char *symname)
554 {
555 struct symbol *sym, *func;
556 unsigned long off, end;
557 struct reloc *reloc;
558 int idx;
559
560 sym = find_symbol_by_name(file->elf, symname);
561 if (!sym)
562 return 0;
563
564 off = sym->offset;
565 end = off + sym->len;
566 for (;;) {
567 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
568 if (!reloc)
569 break;
570
571 func = reloc->sym;
572 if (func->type == STT_SECTION)
573 func = find_symbol_by_offset(reloc->sym->sec,
574 reloc_addend(reloc));
575
576 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
577
578 objtool_pv_add(file, idx, func);
579
580 off = reloc_offset(reloc) + 1;
581 if (off > end)
582 break;
583 }
584
585 return 0;
586 }
587
588 /*
589 * Allocate and initialize file->pv_ops[].
590 */
init_pv_ops(struct objtool_file * file)591 static int init_pv_ops(struct objtool_file *file)
592 {
593 static const char *pv_ops_tables[] = {
594 "pv_ops",
595 "xen_cpu_ops",
596 "xen_irq_ops",
597 "xen_mmu_ops",
598 NULL,
599 };
600 const char *pv_ops;
601 struct symbol *sym;
602 int idx, nr;
603
604 if (!opts.noinstr)
605 return 0;
606
607 file->pv_ops = NULL;
608
609 sym = find_symbol_by_name(file->elf, "pv_ops");
610 if (!sym)
611 return 0;
612
613 nr = sym->len / sizeof(unsigned long);
614 file->pv_ops = calloc(sizeof(struct pv_state), nr);
615 if (!file->pv_ops)
616 return -1;
617
618 for (idx = 0; idx < nr; idx++)
619 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
620
621 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
622 add_pv_ops(file, pv_ops);
623
624 return 0;
625 }
626
create_static_call_sections(struct objtool_file * file)627 static int create_static_call_sections(struct objtool_file *file)
628 {
629 struct static_call_site *site;
630 struct section *sec;
631 struct instruction *insn;
632 struct symbol *key_sym;
633 char *key_name, *tmp;
634 int idx;
635
636 sec = find_section_by_name(file->elf, ".static_call_sites");
637 if (sec) {
638 INIT_LIST_HEAD(&file->static_call_list);
639 WARN("file already has .static_call_sites section, skipping");
640 return 0;
641 }
642
643 if (list_empty(&file->static_call_list))
644 return 0;
645
646 idx = 0;
647 list_for_each_entry(insn, &file->static_call_list, call_node)
648 idx++;
649
650 sec = elf_create_section_pair(file->elf, ".static_call_sites",
651 sizeof(*site), idx, idx * 2);
652 if (!sec)
653 return -1;
654
655 /* Allow modules to modify the low bits of static_call_site::key */
656 sec->sh.sh_flags |= SHF_WRITE;
657
658 idx = 0;
659 list_for_each_entry(insn, &file->static_call_list, call_node) {
660
661 /* populate reloc for 'addr' */
662 if (!elf_init_reloc_text_sym(file->elf, sec,
663 idx * sizeof(*site), idx * 2,
664 insn->sec, insn->offset))
665 return -1;
666
667 /* find key symbol */
668 key_name = strdup(insn_call_dest(insn)->name);
669 if (!key_name) {
670 perror("strdup");
671 return -1;
672 }
673 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
674 STATIC_CALL_TRAMP_PREFIX_LEN)) {
675 WARN("static_call: trampoline name malformed: %s", key_name);
676 free(key_name);
677 return -1;
678 }
679 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
680 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
681
682 key_sym = find_symbol_by_name(file->elf, tmp);
683 if (!key_sym) {
684 if (!opts.module) {
685 WARN("static_call: can't find static_call_key symbol: %s", tmp);
686 free(key_name);
687 return -1;
688 }
689
690 /*
691 * For modules(), the key might not be exported, which
692 * means the module can make static calls but isn't
693 * allowed to change them.
694 *
695 * In that case we temporarily set the key to be the
696 * trampoline address. This is fixed up in
697 * static_call_add_module().
698 */
699 key_sym = insn_call_dest(insn);
700 }
701 free(key_name);
702
703 /* populate reloc for 'key' */
704 if (!elf_init_reloc_data_sym(file->elf, sec,
705 idx * sizeof(*site) + 4,
706 (idx * 2) + 1, key_sym,
707 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
708 return -1;
709
710 idx++;
711 }
712
713 return 0;
714 }
715
create_retpoline_sites_sections(struct objtool_file * file)716 static int create_retpoline_sites_sections(struct objtool_file *file)
717 {
718 struct instruction *insn;
719 struct section *sec;
720 int idx;
721
722 sec = find_section_by_name(file->elf, ".retpoline_sites");
723 if (sec) {
724 WARN("file already has .retpoline_sites, skipping");
725 return 0;
726 }
727
728 idx = 0;
729 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
730 idx++;
731
732 if (!idx)
733 return 0;
734
735 sec = elf_create_section_pair(file->elf, ".retpoline_sites",
736 sizeof(int), idx, idx);
737 if (!sec)
738 return -1;
739
740 idx = 0;
741 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
742
743 if (!elf_init_reloc_text_sym(file->elf, sec,
744 idx * sizeof(int), idx,
745 insn->sec, insn->offset))
746 return -1;
747
748 idx++;
749 }
750
751 return 0;
752 }
753
create_return_sites_sections(struct objtool_file * file)754 static int create_return_sites_sections(struct objtool_file *file)
755 {
756 struct instruction *insn;
757 struct section *sec;
758 int idx;
759
760 sec = find_section_by_name(file->elf, ".return_sites");
761 if (sec) {
762 WARN("file already has .return_sites, skipping");
763 return 0;
764 }
765
766 idx = 0;
767 list_for_each_entry(insn, &file->return_thunk_list, call_node)
768 idx++;
769
770 if (!idx)
771 return 0;
772
773 sec = elf_create_section_pair(file->elf, ".return_sites",
774 sizeof(int), idx, idx);
775 if (!sec)
776 return -1;
777
778 idx = 0;
779 list_for_each_entry(insn, &file->return_thunk_list, call_node) {
780
781 if (!elf_init_reloc_text_sym(file->elf, sec,
782 idx * sizeof(int), idx,
783 insn->sec, insn->offset))
784 return -1;
785
786 idx++;
787 }
788
789 return 0;
790 }
791
create_ibt_endbr_seal_sections(struct objtool_file * file)792 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
793 {
794 struct instruction *insn;
795 struct section *sec;
796 int idx;
797
798 sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
799 if (sec) {
800 WARN("file already has .ibt_endbr_seal, skipping");
801 return 0;
802 }
803
804 idx = 0;
805 list_for_each_entry(insn, &file->endbr_list, call_node)
806 idx++;
807
808 if (opts.stats) {
809 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
810 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
811 printf("ibt: superfluous ENDBR: %d\n", idx);
812 }
813
814 if (!idx)
815 return 0;
816
817 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
818 sizeof(int), idx, idx);
819 if (!sec)
820 return -1;
821
822 idx = 0;
823 list_for_each_entry(insn, &file->endbr_list, call_node) {
824
825 int *site = (int *)sec->data->d_buf + idx;
826 struct symbol *sym = insn->sym;
827 *site = 0;
828
829 if (opts.module && sym && sym->type == STT_FUNC &&
830 insn->offset == sym->offset &&
831 (!strcmp(sym->name, "init_module") ||
832 !strcmp(sym->name, "cleanup_module")))
833 WARN("%s(): not an indirect call target", sym->name);
834
835 if (!elf_init_reloc_text_sym(file->elf, sec,
836 idx * sizeof(int), idx,
837 insn->sec, insn->offset))
838 return -1;
839
840 idx++;
841 }
842
843 return 0;
844 }
845
create_cfi_sections(struct objtool_file * file)846 static int create_cfi_sections(struct objtool_file *file)
847 {
848 struct section *sec;
849 struct symbol *sym;
850 int idx;
851
852 sec = find_section_by_name(file->elf, ".cfi_sites");
853 if (sec) {
854 INIT_LIST_HEAD(&file->call_list);
855 WARN("file already has .cfi_sites section, skipping");
856 return 0;
857 }
858
859 idx = 0;
860 for_each_sym(file, sym) {
861 if (sym->type != STT_FUNC)
862 continue;
863
864 if (strncmp(sym->name, "__cfi_", 6))
865 continue;
866
867 idx++;
868 }
869
870 sec = elf_create_section_pair(file->elf, ".cfi_sites",
871 sizeof(unsigned int), idx, idx);
872 if (!sec)
873 return -1;
874
875 idx = 0;
876 for_each_sym(file, sym) {
877 if (sym->type != STT_FUNC)
878 continue;
879
880 if (strncmp(sym->name, "__cfi_", 6))
881 continue;
882
883 if (!elf_init_reloc_text_sym(file->elf, sec,
884 idx * sizeof(unsigned int), idx,
885 sym->sec, sym->offset))
886 return -1;
887
888 idx++;
889 }
890
891 return 0;
892 }
893
create_mcount_loc_sections(struct objtool_file * file)894 static int create_mcount_loc_sections(struct objtool_file *file)
895 {
896 size_t addr_size = elf_addr_size(file->elf);
897 struct instruction *insn;
898 struct section *sec;
899 int idx;
900
901 sec = find_section_by_name(file->elf, "__mcount_loc");
902 if (sec) {
903 INIT_LIST_HEAD(&file->mcount_loc_list);
904 WARN("file already has __mcount_loc section, skipping");
905 return 0;
906 }
907
908 if (list_empty(&file->mcount_loc_list))
909 return 0;
910
911 idx = 0;
912 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
913 idx++;
914
915 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
916 idx, idx);
917 if (!sec)
918 return -1;
919
920 sec->sh.sh_addralign = addr_size;
921
922 idx = 0;
923 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
924
925 struct reloc *reloc;
926
927 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
928 insn->sec, insn->offset);
929 if (!reloc)
930 return -1;
931
932 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
933
934 idx++;
935 }
936
937 return 0;
938 }
939
create_direct_call_sections(struct objtool_file * file)940 static int create_direct_call_sections(struct objtool_file *file)
941 {
942 struct instruction *insn;
943 struct section *sec;
944 int idx;
945
946 sec = find_section_by_name(file->elf, ".call_sites");
947 if (sec) {
948 INIT_LIST_HEAD(&file->call_list);
949 WARN("file already has .call_sites section, skipping");
950 return 0;
951 }
952
953 if (list_empty(&file->call_list))
954 return 0;
955
956 idx = 0;
957 list_for_each_entry(insn, &file->call_list, call_node)
958 idx++;
959
960 sec = elf_create_section_pair(file->elf, ".call_sites",
961 sizeof(unsigned int), idx, idx);
962 if (!sec)
963 return -1;
964
965 idx = 0;
966 list_for_each_entry(insn, &file->call_list, call_node) {
967
968 if (!elf_init_reloc_text_sym(file->elf, sec,
969 idx * sizeof(unsigned int), idx,
970 insn->sec, insn->offset))
971 return -1;
972
973 idx++;
974 }
975
976 return 0;
977 }
978
979 /*
980 * Warnings shouldn't be reported for ignored functions.
981 */
add_ignores(struct objtool_file * file)982 static void add_ignores(struct objtool_file *file)
983 {
984 struct instruction *insn;
985 struct section *rsec;
986 struct symbol *func;
987 struct reloc *reloc;
988
989 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
990 if (!rsec)
991 return;
992
993 for_each_reloc(rsec, reloc) {
994 switch (reloc->sym->type) {
995 case STT_FUNC:
996 func = reloc->sym;
997 break;
998
999 case STT_SECTION:
1000 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1001 if (!func)
1002 continue;
1003 break;
1004
1005 default:
1006 WARN("unexpected relocation symbol type in %s: %d",
1007 rsec->name, reloc->sym->type);
1008 continue;
1009 }
1010
1011 func_for_each_insn(file, func, insn)
1012 insn->ignore = true;
1013 }
1014 }
1015
1016 /*
1017 * This is a whitelist of functions that is allowed to be called with AC set.
1018 * The list is meant to be minimal and only contains compiler instrumentation
1019 * ABI and a few functions used to implement *_{to,from}_user() functions.
1020 *
1021 * These functions must not directly change AC, but may PUSHF/POPF.
1022 */
1023 static const char *uaccess_safe_builtin[] = {
1024 /* KASAN */
1025 "kasan_report",
1026 "kasan_check_range",
1027 /* KASAN out-of-line */
1028 "__asan_loadN_noabort",
1029 "__asan_load1_noabort",
1030 "__asan_load2_noabort",
1031 "__asan_load4_noabort",
1032 "__asan_load8_noabort",
1033 "__asan_load16_noabort",
1034 "__asan_storeN_noabort",
1035 "__asan_store1_noabort",
1036 "__asan_store2_noabort",
1037 "__asan_store4_noabort",
1038 "__asan_store8_noabort",
1039 "__asan_store16_noabort",
1040 "__kasan_check_read",
1041 "__kasan_check_write",
1042 /* KASAN in-line */
1043 "__asan_report_load_n_noabort",
1044 "__asan_report_load1_noabort",
1045 "__asan_report_load2_noabort",
1046 "__asan_report_load4_noabort",
1047 "__asan_report_load8_noabort",
1048 "__asan_report_load16_noabort",
1049 "__asan_report_store_n_noabort",
1050 "__asan_report_store1_noabort",
1051 "__asan_report_store2_noabort",
1052 "__asan_report_store4_noabort",
1053 "__asan_report_store8_noabort",
1054 "__asan_report_store16_noabort",
1055 /* KCSAN */
1056 "__kcsan_check_access",
1057 "__kcsan_mb",
1058 "__kcsan_wmb",
1059 "__kcsan_rmb",
1060 "__kcsan_release",
1061 "kcsan_found_watchpoint",
1062 "kcsan_setup_watchpoint",
1063 "kcsan_check_scoped_accesses",
1064 "kcsan_disable_current",
1065 "kcsan_enable_current_nowarn",
1066 /* KCSAN/TSAN */
1067 "__tsan_func_entry",
1068 "__tsan_func_exit",
1069 "__tsan_read_range",
1070 "__tsan_write_range",
1071 "__tsan_read1",
1072 "__tsan_read2",
1073 "__tsan_read4",
1074 "__tsan_read8",
1075 "__tsan_read16",
1076 "__tsan_write1",
1077 "__tsan_write2",
1078 "__tsan_write4",
1079 "__tsan_write8",
1080 "__tsan_write16",
1081 "__tsan_read_write1",
1082 "__tsan_read_write2",
1083 "__tsan_read_write4",
1084 "__tsan_read_write8",
1085 "__tsan_read_write16",
1086 "__tsan_volatile_read1",
1087 "__tsan_volatile_read2",
1088 "__tsan_volatile_read4",
1089 "__tsan_volatile_read8",
1090 "__tsan_volatile_read16",
1091 "__tsan_volatile_write1",
1092 "__tsan_volatile_write2",
1093 "__tsan_volatile_write4",
1094 "__tsan_volatile_write8",
1095 "__tsan_volatile_write16",
1096 "__tsan_atomic8_load",
1097 "__tsan_atomic16_load",
1098 "__tsan_atomic32_load",
1099 "__tsan_atomic64_load",
1100 "__tsan_atomic8_store",
1101 "__tsan_atomic16_store",
1102 "__tsan_atomic32_store",
1103 "__tsan_atomic64_store",
1104 "__tsan_atomic8_exchange",
1105 "__tsan_atomic16_exchange",
1106 "__tsan_atomic32_exchange",
1107 "__tsan_atomic64_exchange",
1108 "__tsan_atomic8_fetch_add",
1109 "__tsan_atomic16_fetch_add",
1110 "__tsan_atomic32_fetch_add",
1111 "__tsan_atomic64_fetch_add",
1112 "__tsan_atomic8_fetch_sub",
1113 "__tsan_atomic16_fetch_sub",
1114 "__tsan_atomic32_fetch_sub",
1115 "__tsan_atomic64_fetch_sub",
1116 "__tsan_atomic8_fetch_and",
1117 "__tsan_atomic16_fetch_and",
1118 "__tsan_atomic32_fetch_and",
1119 "__tsan_atomic64_fetch_and",
1120 "__tsan_atomic8_fetch_or",
1121 "__tsan_atomic16_fetch_or",
1122 "__tsan_atomic32_fetch_or",
1123 "__tsan_atomic64_fetch_or",
1124 "__tsan_atomic8_fetch_xor",
1125 "__tsan_atomic16_fetch_xor",
1126 "__tsan_atomic32_fetch_xor",
1127 "__tsan_atomic64_fetch_xor",
1128 "__tsan_atomic8_fetch_nand",
1129 "__tsan_atomic16_fetch_nand",
1130 "__tsan_atomic32_fetch_nand",
1131 "__tsan_atomic64_fetch_nand",
1132 "__tsan_atomic8_compare_exchange_strong",
1133 "__tsan_atomic16_compare_exchange_strong",
1134 "__tsan_atomic32_compare_exchange_strong",
1135 "__tsan_atomic64_compare_exchange_strong",
1136 "__tsan_atomic8_compare_exchange_weak",
1137 "__tsan_atomic16_compare_exchange_weak",
1138 "__tsan_atomic32_compare_exchange_weak",
1139 "__tsan_atomic64_compare_exchange_weak",
1140 "__tsan_atomic8_compare_exchange_val",
1141 "__tsan_atomic16_compare_exchange_val",
1142 "__tsan_atomic32_compare_exchange_val",
1143 "__tsan_atomic64_compare_exchange_val",
1144 "__tsan_atomic_thread_fence",
1145 "__tsan_atomic_signal_fence",
1146 "__tsan_unaligned_read16",
1147 "__tsan_unaligned_write16",
1148 /* KCOV */
1149 "write_comp_data",
1150 "check_kcov_mode",
1151 "__sanitizer_cov_trace_pc",
1152 "__sanitizer_cov_trace_const_cmp1",
1153 "__sanitizer_cov_trace_const_cmp2",
1154 "__sanitizer_cov_trace_const_cmp4",
1155 "__sanitizer_cov_trace_const_cmp8",
1156 "__sanitizer_cov_trace_cmp1",
1157 "__sanitizer_cov_trace_cmp2",
1158 "__sanitizer_cov_trace_cmp4",
1159 "__sanitizer_cov_trace_cmp8",
1160 "__sanitizer_cov_trace_switch",
1161 /* KMSAN */
1162 "kmsan_copy_to_user",
1163 "kmsan_disable_current",
1164 "kmsan_enable_current",
1165 "kmsan_report",
1166 "kmsan_unpoison_entry_regs",
1167 "kmsan_unpoison_memory",
1168 "__msan_chain_origin",
1169 "__msan_get_context_state",
1170 "__msan_instrument_asm_store",
1171 "__msan_metadata_ptr_for_load_1",
1172 "__msan_metadata_ptr_for_load_2",
1173 "__msan_metadata_ptr_for_load_4",
1174 "__msan_metadata_ptr_for_load_8",
1175 "__msan_metadata_ptr_for_load_n",
1176 "__msan_metadata_ptr_for_store_1",
1177 "__msan_metadata_ptr_for_store_2",
1178 "__msan_metadata_ptr_for_store_4",
1179 "__msan_metadata_ptr_for_store_8",
1180 "__msan_metadata_ptr_for_store_n",
1181 "__msan_poison_alloca",
1182 "__msan_warning",
1183 /* UBSAN */
1184 "ubsan_type_mismatch_common",
1185 "__ubsan_handle_type_mismatch",
1186 "__ubsan_handle_type_mismatch_v1",
1187 "__ubsan_handle_shift_out_of_bounds",
1188 "__ubsan_handle_load_invalid_value",
1189 /* STACKLEAK */
1190 "stackleak_track_stack",
1191 /* misc */
1192 "csum_partial_copy_generic",
1193 "copy_mc_fragile",
1194 "copy_mc_fragile_handle_tail",
1195 "copy_mc_enhanced_fast_string",
1196 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1197 "rep_stos_alternative",
1198 "rep_movs_alternative",
1199 "__copy_user_nocache",
1200 NULL
1201 };
1202
add_uaccess_safe(struct objtool_file * file)1203 static void add_uaccess_safe(struct objtool_file *file)
1204 {
1205 struct symbol *func;
1206 const char **name;
1207
1208 if (!opts.uaccess)
1209 return;
1210
1211 for (name = uaccess_safe_builtin; *name; name++) {
1212 func = find_symbol_by_name(file->elf, *name);
1213 if (!func)
1214 continue;
1215
1216 func->uaccess_safe = true;
1217 }
1218 }
1219
1220 /*
1221 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1222 * will be added to the .retpoline_sites section.
1223 */
arch_is_retpoline(struct symbol * sym)1224 __weak bool arch_is_retpoline(struct symbol *sym)
1225 {
1226 return false;
1227 }
1228
1229 /*
1230 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1231 * will be added to the .return_sites section.
1232 */
arch_is_rethunk(struct symbol * sym)1233 __weak bool arch_is_rethunk(struct symbol *sym)
1234 {
1235 return false;
1236 }
1237
1238 /*
1239 * Symbols that are embedded inside other instructions, because sometimes crazy
1240 * code exists. These are mostly ignored for validation purposes.
1241 */
arch_is_embedded_insn(struct symbol * sym)1242 __weak bool arch_is_embedded_insn(struct symbol *sym)
1243 {
1244 return false;
1245 }
1246
insn_reloc(struct objtool_file * file,struct instruction * insn)1247 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1248 {
1249 struct reloc *reloc;
1250
1251 if (insn->no_reloc)
1252 return NULL;
1253
1254 if (!file)
1255 return NULL;
1256
1257 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1258 insn->offset, insn->len);
1259 if (!reloc) {
1260 insn->no_reloc = 1;
1261 return NULL;
1262 }
1263
1264 return reloc;
1265 }
1266
remove_insn_ops(struct instruction * insn)1267 static void remove_insn_ops(struct instruction *insn)
1268 {
1269 struct stack_op *op, *next;
1270
1271 for (op = insn->stack_ops; op; op = next) {
1272 next = op->next;
1273 free(op);
1274 }
1275 insn->stack_ops = NULL;
1276 }
1277
annotate_call_site(struct objtool_file * file,struct instruction * insn,bool sibling)1278 static void annotate_call_site(struct objtool_file *file,
1279 struct instruction *insn, bool sibling)
1280 {
1281 struct reloc *reloc = insn_reloc(file, insn);
1282 struct symbol *sym = insn_call_dest(insn);
1283
1284 if (!sym)
1285 sym = reloc->sym;
1286
1287 if (sym->static_call_tramp) {
1288 list_add_tail(&insn->call_node, &file->static_call_list);
1289 return;
1290 }
1291
1292 if (sym->retpoline_thunk) {
1293 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1294 return;
1295 }
1296
1297 /*
1298 * Many compilers cannot disable KCOV or sanitizer calls with a function
1299 * attribute so they need a little help, NOP out any such calls from
1300 * noinstr text.
1301 */
1302 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1303 if (reloc)
1304 set_reloc_type(file->elf, reloc, R_NONE);
1305
1306 elf_write_insn(file->elf, insn->sec,
1307 insn->offset, insn->len,
1308 sibling ? arch_ret_insn(insn->len)
1309 : arch_nop_insn(insn->len));
1310
1311 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1312
1313 if (sibling) {
1314 /*
1315 * We've replaced the tail-call JMP insn by two new
1316 * insn: RET; INT3, except we only have a single struct
1317 * insn here. Mark it retpoline_safe to avoid the SLS
1318 * warning, instead of adding another insn.
1319 */
1320 insn->retpoline_safe = true;
1321 }
1322
1323 return;
1324 }
1325
1326 if (opts.mcount && sym->fentry) {
1327 if (sibling)
1328 WARN_INSN(insn, "tail call to __fentry__ !?!?");
1329 if (opts.mnop) {
1330 if (reloc)
1331 set_reloc_type(file->elf, reloc, R_NONE);
1332
1333 elf_write_insn(file->elf, insn->sec,
1334 insn->offset, insn->len,
1335 arch_nop_insn(insn->len));
1336
1337 insn->type = INSN_NOP;
1338 }
1339
1340 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1341 return;
1342 }
1343
1344 if (insn->type == INSN_CALL && !insn->sec->init &&
1345 !insn->_call_dest->embedded_insn)
1346 list_add_tail(&insn->call_node, &file->call_list);
1347
1348 if (!sibling && dead_end_function(file, sym))
1349 insn->dead_end = true;
1350 }
1351
add_call_dest(struct objtool_file * file,struct instruction * insn,struct symbol * dest,bool sibling)1352 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1353 struct symbol *dest, bool sibling)
1354 {
1355 insn->_call_dest = dest;
1356 if (!dest)
1357 return;
1358
1359 /*
1360 * Whatever stack impact regular CALLs have, should be undone
1361 * by the RETURN of the called function.
1362 *
1363 * Annotated intra-function calls retain the stack_ops but
1364 * are converted to JUMP, see read_intra_function_calls().
1365 */
1366 remove_insn_ops(insn);
1367
1368 annotate_call_site(file, insn, sibling);
1369 }
1370
add_retpoline_call(struct objtool_file * file,struct instruction * insn)1371 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1372 {
1373 /*
1374 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1375 * so convert them accordingly.
1376 */
1377 switch (insn->type) {
1378 case INSN_CALL:
1379 insn->type = INSN_CALL_DYNAMIC;
1380 break;
1381 case INSN_JUMP_UNCONDITIONAL:
1382 insn->type = INSN_JUMP_DYNAMIC;
1383 break;
1384 case INSN_JUMP_CONDITIONAL:
1385 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1386 break;
1387 default:
1388 return;
1389 }
1390
1391 insn->retpoline_safe = true;
1392
1393 /*
1394 * Whatever stack impact regular CALLs have, should be undone
1395 * by the RETURN of the called function.
1396 *
1397 * Annotated intra-function calls retain the stack_ops but
1398 * are converted to JUMP, see read_intra_function_calls().
1399 */
1400 remove_insn_ops(insn);
1401
1402 annotate_call_site(file, insn, false);
1403 }
1404
add_return_call(struct objtool_file * file,struct instruction * insn,bool add)1405 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1406 {
1407 /*
1408 * Return thunk tail calls are really just returns in disguise,
1409 * so convert them accordingly.
1410 */
1411 insn->type = INSN_RETURN;
1412 insn->retpoline_safe = true;
1413
1414 if (add)
1415 list_add_tail(&insn->call_node, &file->return_thunk_list);
1416 }
1417
is_first_func_insn(struct objtool_file * file,struct instruction * insn,struct symbol * sym)1418 static bool is_first_func_insn(struct objtool_file *file,
1419 struct instruction *insn, struct symbol *sym)
1420 {
1421 if (insn->offset == sym->offset)
1422 return true;
1423
1424 /* Allow direct CALL/JMP past ENDBR */
1425 if (opts.ibt) {
1426 struct instruction *prev = prev_insn_same_sym(file, insn);
1427
1428 if (prev && prev->type == INSN_ENDBR &&
1429 insn->offset == sym->offset + prev->len)
1430 return true;
1431 }
1432
1433 return false;
1434 }
1435
1436 /*
1437 * A sibling call is a tail-call to another symbol -- to differentiate from a
1438 * recursive tail-call which is to the same symbol.
1439 */
jump_is_sibling_call(struct objtool_file * file,struct instruction * from,struct instruction * to)1440 static bool jump_is_sibling_call(struct objtool_file *file,
1441 struct instruction *from, struct instruction *to)
1442 {
1443 struct symbol *fs = from->sym;
1444 struct symbol *ts = to->sym;
1445
1446 /* Not a sibling call if from/to a symbol hole */
1447 if (!fs || !ts)
1448 return false;
1449
1450 /* Not a sibling call if not targeting the start of a symbol. */
1451 if (!is_first_func_insn(file, to, ts))
1452 return false;
1453
1454 /* Disallow sibling calls into STT_NOTYPE */
1455 if (ts->type == STT_NOTYPE)
1456 return false;
1457
1458 /* Must not be self to be a sibling */
1459 return fs->pfunc != ts->pfunc;
1460 }
1461
1462 /*
1463 * Find the destination instructions for all jumps.
1464 */
add_jump_destinations(struct objtool_file * file)1465 static int add_jump_destinations(struct objtool_file *file)
1466 {
1467 struct instruction *insn, *jump_dest;
1468 struct reloc *reloc;
1469 struct section *dest_sec;
1470 unsigned long dest_off;
1471
1472 for_each_insn(file, insn) {
1473 if (insn->jump_dest) {
1474 /*
1475 * handle_group_alt() may have previously set
1476 * 'jump_dest' for some alternatives.
1477 */
1478 continue;
1479 }
1480 if (!is_static_jump(insn))
1481 continue;
1482
1483 reloc = insn_reloc(file, insn);
1484 if (!reloc) {
1485 dest_sec = insn->sec;
1486 dest_off = arch_jump_destination(insn);
1487 } else if (reloc->sym->type == STT_SECTION) {
1488 dest_sec = reloc->sym->sec;
1489 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1490 } else if (reloc->sym->retpoline_thunk) {
1491 add_retpoline_call(file, insn);
1492 continue;
1493 } else if (reloc->sym->return_thunk) {
1494 add_return_call(file, insn, true);
1495 continue;
1496 } else if (insn_func(insn)) {
1497 /*
1498 * External sibling call or internal sibling call with
1499 * STT_FUNC reloc.
1500 */
1501 add_call_dest(file, insn, reloc->sym, true);
1502 continue;
1503 } else if (reloc->sym->sec->idx) {
1504 dest_sec = reloc->sym->sec;
1505 dest_off = reloc->sym->sym.st_value +
1506 arch_dest_reloc_offset(reloc_addend(reloc));
1507 } else {
1508 /* non-func asm code jumping to another file */
1509 continue;
1510 }
1511
1512 jump_dest = find_insn(file, dest_sec, dest_off);
1513 if (!jump_dest) {
1514 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1515
1516 /*
1517 * This is a special case for retbleed_untrain_ret().
1518 * It jumps to __x86_return_thunk(), but objtool
1519 * can't find the thunk's starting RET
1520 * instruction, because the RET is also in the
1521 * middle of another instruction. Objtool only
1522 * knows about the outer instruction.
1523 */
1524 if (sym && sym->embedded_insn) {
1525 add_return_call(file, insn, false);
1526 continue;
1527 }
1528
1529 WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1530 dest_sec->name, dest_off);
1531 return -1;
1532 }
1533
1534 /*
1535 * An intra-TU jump in retpoline.o might not have a relocation
1536 * for its jump dest, in which case the above
1537 * add_{retpoline,return}_call() didn't happen.
1538 */
1539 if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1540 if (jump_dest->sym->retpoline_thunk) {
1541 add_retpoline_call(file, insn);
1542 continue;
1543 }
1544 if (jump_dest->sym->return_thunk) {
1545 add_return_call(file, insn, true);
1546 continue;
1547 }
1548 }
1549
1550 /*
1551 * Cross-function jump.
1552 */
1553 if (insn_func(insn) && insn_func(jump_dest) &&
1554 insn_func(insn) != insn_func(jump_dest)) {
1555
1556 /*
1557 * For GCC 8+, create parent/child links for any cold
1558 * subfunctions. This is _mostly_ redundant with a
1559 * similar initialization in read_symbols().
1560 *
1561 * If a function has aliases, we want the *first* such
1562 * function in the symbol table to be the subfunction's
1563 * parent. In that case we overwrite the
1564 * initialization done in read_symbols().
1565 *
1566 * However this code can't completely replace the
1567 * read_symbols() code because this doesn't detect the
1568 * case where the parent function's only reference to a
1569 * subfunction is through a jump table.
1570 */
1571 if (!strstr(insn_func(insn)->name, ".cold") &&
1572 strstr(insn_func(jump_dest)->name, ".cold")) {
1573 insn_func(insn)->cfunc = insn_func(jump_dest);
1574 insn_func(jump_dest)->pfunc = insn_func(insn);
1575 }
1576 }
1577
1578 if (jump_is_sibling_call(file, insn, jump_dest)) {
1579 /*
1580 * Internal sibling call without reloc or with
1581 * STT_SECTION reloc.
1582 */
1583 add_call_dest(file, insn, insn_func(jump_dest), true);
1584 continue;
1585 }
1586
1587 insn->jump_dest = jump_dest;
1588 }
1589
1590 return 0;
1591 }
1592
find_call_destination(struct section * sec,unsigned long offset)1593 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1594 {
1595 struct symbol *call_dest;
1596
1597 call_dest = find_func_by_offset(sec, offset);
1598 if (!call_dest)
1599 call_dest = find_symbol_by_offset(sec, offset);
1600
1601 return call_dest;
1602 }
1603
1604 /*
1605 * Find the destination instructions for all calls.
1606 */
add_call_destinations(struct objtool_file * file)1607 static int add_call_destinations(struct objtool_file *file)
1608 {
1609 struct instruction *insn;
1610 unsigned long dest_off;
1611 struct symbol *dest;
1612 struct reloc *reloc;
1613
1614 for_each_insn(file, insn) {
1615 if (insn->type != INSN_CALL)
1616 continue;
1617
1618 reloc = insn_reloc(file, insn);
1619 if (!reloc) {
1620 dest_off = arch_jump_destination(insn);
1621 dest = find_call_destination(insn->sec, dest_off);
1622
1623 add_call_dest(file, insn, dest, false);
1624
1625 if (insn->ignore)
1626 continue;
1627
1628 if (!insn_call_dest(insn)) {
1629 WARN_INSN(insn, "unannotated intra-function call");
1630 return -1;
1631 }
1632
1633 if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1634 WARN_INSN(insn, "unsupported call to non-function");
1635 return -1;
1636 }
1637
1638 } else if (reloc->sym->type == STT_SECTION) {
1639 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1640 dest = find_call_destination(reloc->sym->sec, dest_off);
1641 if (!dest) {
1642 WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1643 reloc->sym->sec->name, dest_off);
1644 return -1;
1645 }
1646
1647 add_call_dest(file, insn, dest, false);
1648
1649 } else if (reloc->sym->retpoline_thunk) {
1650 add_retpoline_call(file, insn);
1651
1652 } else
1653 add_call_dest(file, insn, reloc->sym, false);
1654 }
1655
1656 return 0;
1657 }
1658
1659 /*
1660 * The .alternatives section requires some extra special care over and above
1661 * other special sections because alternatives are patched in place.
1662 */
handle_group_alt(struct objtool_file * file,struct special_alt * special_alt,struct instruction * orig_insn,struct instruction ** new_insn)1663 static int handle_group_alt(struct objtool_file *file,
1664 struct special_alt *special_alt,
1665 struct instruction *orig_insn,
1666 struct instruction **new_insn)
1667 {
1668 struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1669 struct alt_group *orig_alt_group, *new_alt_group;
1670 unsigned long dest_off;
1671
1672 orig_alt_group = orig_insn->alt_group;
1673 if (!orig_alt_group) {
1674 struct instruction *last_orig_insn = NULL;
1675
1676 orig_alt_group = malloc(sizeof(*orig_alt_group));
1677 if (!orig_alt_group) {
1678 WARN("malloc failed");
1679 return -1;
1680 }
1681 orig_alt_group->cfi = calloc(special_alt->orig_len,
1682 sizeof(struct cfi_state *));
1683 if (!orig_alt_group->cfi) {
1684 WARN("calloc failed");
1685 return -1;
1686 }
1687
1688 insn = orig_insn;
1689 sec_for_each_insn_from(file, insn) {
1690 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1691 break;
1692
1693 insn->alt_group = orig_alt_group;
1694 last_orig_insn = insn;
1695 }
1696 orig_alt_group->orig_group = NULL;
1697 orig_alt_group->first_insn = orig_insn;
1698 orig_alt_group->last_insn = last_orig_insn;
1699 orig_alt_group->nop = NULL;
1700 } else {
1701 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1702 orig_alt_group->first_insn->offset != special_alt->orig_len) {
1703 WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1704 orig_alt_group->last_insn->offset +
1705 orig_alt_group->last_insn->len -
1706 orig_alt_group->first_insn->offset,
1707 special_alt->orig_len);
1708 return -1;
1709 }
1710 }
1711
1712 new_alt_group = malloc(sizeof(*new_alt_group));
1713 if (!new_alt_group) {
1714 WARN("malloc failed");
1715 return -1;
1716 }
1717
1718 if (special_alt->new_len < special_alt->orig_len) {
1719 /*
1720 * Insert a fake nop at the end to make the replacement
1721 * alt_group the same size as the original. This is needed to
1722 * allow propagate_alt_cfi() to do its magic. When the last
1723 * instruction affects the stack, the instruction after it (the
1724 * nop) will propagate the new state to the shared CFI array.
1725 */
1726 nop = malloc(sizeof(*nop));
1727 if (!nop) {
1728 WARN("malloc failed");
1729 return -1;
1730 }
1731 memset(nop, 0, sizeof(*nop));
1732
1733 nop->sec = special_alt->new_sec;
1734 nop->offset = special_alt->new_off + special_alt->new_len;
1735 nop->len = special_alt->orig_len - special_alt->new_len;
1736 nop->type = INSN_NOP;
1737 nop->sym = orig_insn->sym;
1738 nop->alt_group = new_alt_group;
1739 nop->ignore = orig_insn->ignore_alts;
1740 }
1741
1742 if (!special_alt->new_len) {
1743 *new_insn = nop;
1744 goto end;
1745 }
1746
1747 insn = *new_insn;
1748 sec_for_each_insn_from(file, insn) {
1749 struct reloc *alt_reloc;
1750
1751 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1752 break;
1753
1754 last_new_insn = insn;
1755
1756 insn->ignore = orig_insn->ignore_alts;
1757 insn->sym = orig_insn->sym;
1758 insn->alt_group = new_alt_group;
1759
1760 /*
1761 * Since alternative replacement code is copy/pasted by the
1762 * kernel after applying relocations, generally such code can't
1763 * have relative-address relocation references to outside the
1764 * .altinstr_replacement section, unless the arch's
1765 * alternatives code can adjust the relative offsets
1766 * accordingly.
1767 */
1768 alt_reloc = insn_reloc(file, insn);
1769 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1770 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1771
1772 WARN_INSN(insn, "unsupported relocation in alternatives section");
1773 return -1;
1774 }
1775
1776 if (!is_static_jump(insn))
1777 continue;
1778
1779 if (!insn->immediate)
1780 continue;
1781
1782 dest_off = arch_jump_destination(insn);
1783 if (dest_off == special_alt->new_off + special_alt->new_len) {
1784 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1785 if (!insn->jump_dest) {
1786 WARN_INSN(insn, "can't find alternative jump destination");
1787 return -1;
1788 }
1789 }
1790 }
1791
1792 if (!last_new_insn) {
1793 WARN_FUNC("can't find last new alternative instruction",
1794 special_alt->new_sec, special_alt->new_off);
1795 return -1;
1796 }
1797
1798 end:
1799 new_alt_group->orig_group = orig_alt_group;
1800 new_alt_group->first_insn = *new_insn;
1801 new_alt_group->last_insn = last_new_insn;
1802 new_alt_group->nop = nop;
1803 new_alt_group->cfi = orig_alt_group->cfi;
1804 return 0;
1805 }
1806
1807 /*
1808 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1809 * If the original instruction is a jump, make the alt entry an effective nop
1810 * by just skipping the original instruction.
1811 */
handle_jump_alt(struct objtool_file * file,struct special_alt * special_alt,struct instruction * orig_insn,struct instruction ** new_insn)1812 static int handle_jump_alt(struct objtool_file *file,
1813 struct special_alt *special_alt,
1814 struct instruction *orig_insn,
1815 struct instruction **new_insn)
1816 {
1817 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1818 orig_insn->type != INSN_NOP) {
1819
1820 WARN_INSN(orig_insn, "unsupported instruction at jump label");
1821 return -1;
1822 }
1823
1824 if (opts.hack_jump_label && special_alt->key_addend & 2) {
1825 struct reloc *reloc = insn_reloc(file, orig_insn);
1826
1827 if (reloc)
1828 set_reloc_type(file->elf, reloc, R_NONE);
1829 elf_write_insn(file->elf, orig_insn->sec,
1830 orig_insn->offset, orig_insn->len,
1831 arch_nop_insn(orig_insn->len));
1832 orig_insn->type = INSN_NOP;
1833 }
1834
1835 if (orig_insn->type == INSN_NOP) {
1836 if (orig_insn->len == 2)
1837 file->jl_nop_short++;
1838 else
1839 file->jl_nop_long++;
1840
1841 return 0;
1842 }
1843
1844 if (orig_insn->len == 2)
1845 file->jl_short++;
1846 else
1847 file->jl_long++;
1848
1849 *new_insn = next_insn_same_sec(file, orig_insn);
1850 return 0;
1851 }
1852
1853 /*
1854 * Read all the special sections which have alternate instructions which can be
1855 * patched in or redirected to at runtime. Each instruction having alternate
1856 * instruction(s) has them added to its insn->alts list, which will be
1857 * traversed in validate_branch().
1858 */
add_special_section_alts(struct objtool_file * file)1859 static int add_special_section_alts(struct objtool_file *file)
1860 {
1861 struct list_head special_alts;
1862 struct instruction *orig_insn, *new_insn;
1863 struct special_alt *special_alt, *tmp;
1864 struct alternative *alt;
1865 int ret;
1866
1867 ret = special_get_alts(file->elf, &special_alts);
1868 if (ret)
1869 return ret;
1870
1871 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1872
1873 orig_insn = find_insn(file, special_alt->orig_sec,
1874 special_alt->orig_off);
1875 if (!orig_insn) {
1876 WARN_FUNC("special: can't find orig instruction",
1877 special_alt->orig_sec, special_alt->orig_off);
1878 ret = -1;
1879 goto out;
1880 }
1881
1882 new_insn = NULL;
1883 if (!special_alt->group || special_alt->new_len) {
1884 new_insn = find_insn(file, special_alt->new_sec,
1885 special_alt->new_off);
1886 if (!new_insn) {
1887 WARN_FUNC("special: can't find new instruction",
1888 special_alt->new_sec,
1889 special_alt->new_off);
1890 ret = -1;
1891 goto out;
1892 }
1893 }
1894
1895 if (special_alt->group) {
1896 if (!special_alt->orig_len) {
1897 WARN_INSN(orig_insn, "empty alternative entry");
1898 continue;
1899 }
1900
1901 ret = handle_group_alt(file, special_alt, orig_insn,
1902 &new_insn);
1903 if (ret)
1904 goto out;
1905 } else if (special_alt->jump_or_nop) {
1906 ret = handle_jump_alt(file, special_alt, orig_insn,
1907 &new_insn);
1908 if (ret)
1909 goto out;
1910 }
1911
1912 alt = malloc(sizeof(*alt));
1913 if (!alt) {
1914 WARN("malloc failed");
1915 ret = -1;
1916 goto out;
1917 }
1918
1919 alt->insn = new_insn;
1920 alt->skip_orig = special_alt->skip_orig;
1921 orig_insn->ignore_alts |= special_alt->skip_alt;
1922 alt->next = orig_insn->alts;
1923 orig_insn->alts = alt;
1924
1925 list_del(&special_alt->list);
1926 free(special_alt);
1927 }
1928
1929 if (opts.stats) {
1930 printf("jl\\\tNOP\tJMP\n");
1931 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1932 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1933 }
1934
1935 out:
1936 return ret;
1937 }
1938
arch_jump_table_sym_offset(struct reloc * reloc,struct reloc * table)1939 __weak unsigned long arch_jump_table_sym_offset(struct reloc *reloc, struct reloc *table)
1940 {
1941 return reloc->sym->offset + reloc_addend(reloc);
1942 }
1943
add_jump_table(struct objtool_file * file,struct instruction * insn,struct reloc * next_table)1944 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
1945 struct reloc *next_table)
1946 {
1947 unsigned long table_size = insn_jump_table_size(insn);
1948 struct symbol *pfunc = insn_func(insn)->pfunc;
1949 struct reloc *table = insn_jump_table(insn);
1950 struct instruction *dest_insn;
1951 unsigned int prev_offset = 0;
1952 struct reloc *reloc = table;
1953 struct alternative *alt;
1954 unsigned long sym_offset;
1955
1956 /*
1957 * Each @reloc is a switch table relocation which points to the target
1958 * instruction.
1959 */
1960 for_each_reloc_from(table->sec, reloc) {
1961
1962 /* Check for the end of the table: */
1963 if (table_size && reloc_offset(reloc) - reloc_offset(table) >= table_size)
1964 break;
1965 if (reloc != table && reloc == next_table)
1966 break;
1967
1968 /* Make sure the table entries are consecutive: */
1969 if (prev_offset && reloc_offset(reloc) != prev_offset + arch_reloc_size(reloc))
1970 break;
1971
1972 sym_offset = arch_jump_table_sym_offset(reloc, table);
1973
1974 /* Detect function pointers from contiguous objects: */
1975 if (reloc->sym->sec == pfunc->sec && sym_offset == pfunc->offset)
1976 break;
1977
1978 /*
1979 * Clang sometimes leaves dangling unused jump table entries
1980 * which point to the end of the function. Ignore them.
1981 */
1982 if (reloc->sym->sec == pfunc->sec &&
1983 sym_offset == pfunc->offset + pfunc->len)
1984 goto next;
1985
1986 dest_insn = find_insn(file, reloc->sym->sec, sym_offset);
1987 if (!dest_insn)
1988 break;
1989
1990 /* Make sure the destination is in the same function: */
1991 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
1992 break;
1993
1994 alt = malloc(sizeof(*alt));
1995 if (!alt) {
1996 WARN("malloc failed");
1997 return -1;
1998 }
1999
2000 alt->insn = dest_insn;
2001 alt->next = insn->alts;
2002 insn->alts = alt;
2003 next:
2004 prev_offset = reloc_offset(reloc);
2005 }
2006
2007 if (!prev_offset) {
2008 WARN_INSN(insn, "can't find switch jump table");
2009 return -1;
2010 }
2011
2012 return 0;
2013 }
2014
2015 /*
2016 * find_jump_table() - Given a dynamic jump, find the switch jump table
2017 * associated with it.
2018 */
find_jump_table(struct objtool_file * file,struct symbol * func,struct instruction * insn)2019 static void find_jump_table(struct objtool_file *file, struct symbol *func,
2020 struct instruction *insn)
2021 {
2022 struct reloc *table_reloc;
2023 struct instruction *dest_insn, *orig_insn = insn;
2024 unsigned long table_size;
2025 unsigned long sym_offset;
2026
2027 /*
2028 * Backward search using the @first_jump_src links, these help avoid
2029 * much of the 'in between' code. Which avoids us getting confused by
2030 * it.
2031 */
2032 for (;
2033 insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2034 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2035
2036 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2037 break;
2038
2039 /* allow small jumps within the range */
2040 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2041 insn->jump_dest &&
2042 (insn->jump_dest->offset <= insn->offset ||
2043 insn->jump_dest->offset > orig_insn->offset))
2044 break;
2045
2046 table_reloc = arch_find_switch_table(file, insn, &table_size);
2047 if (!table_reloc)
2048 continue;
2049
2050 sym_offset = table_reloc->sym->offset + reloc_addend(table_reloc);
2051
2052 dest_insn = find_insn(file, table_reloc->sym->sec, sym_offset);
2053 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2054 continue;
2055
2056 orig_insn->_jump_table = table_reloc;
2057 orig_insn->_jump_table_size = table_size;
2058 break;
2059 }
2060 }
2061
2062 /*
2063 * First pass: Mark the head of each jump table so that in the next pass,
2064 * we know when a given jump table ends and the next one starts.
2065 */
mark_func_jump_tables(struct objtool_file * file,struct symbol * func)2066 static void mark_func_jump_tables(struct objtool_file *file,
2067 struct symbol *func)
2068 {
2069 struct instruction *insn, *last = NULL;
2070
2071 func_for_each_insn(file, func, insn) {
2072 if (!last)
2073 last = insn;
2074
2075 /*
2076 * Store back-pointers for unconditional forward jumps such
2077 * that find_jump_table() can back-track using those and
2078 * avoid some potentially confusing code.
2079 */
2080 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2081 insn->offset > last->offset &&
2082 insn->jump_dest->offset > insn->offset &&
2083 !insn->jump_dest->first_jump_src) {
2084
2085 insn->jump_dest->first_jump_src = insn;
2086 last = insn->jump_dest;
2087 }
2088
2089 if (insn->type != INSN_JUMP_DYNAMIC)
2090 continue;
2091
2092 find_jump_table(file, func, insn);
2093 }
2094 }
2095
add_func_jump_tables(struct objtool_file * file,struct symbol * func)2096 static int add_func_jump_tables(struct objtool_file *file,
2097 struct symbol *func)
2098 {
2099 struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2100 int ret = 0;
2101
2102 func_for_each_insn(file, func, insn) {
2103 if (!insn_jump_table(insn))
2104 continue;
2105
2106 if (!insn_t1) {
2107 insn_t1 = insn;
2108 continue;
2109 }
2110
2111 insn_t2 = insn;
2112
2113 ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2114 if (ret)
2115 return ret;
2116
2117 insn_t1 = insn_t2;
2118 }
2119
2120 if (insn_t1)
2121 ret = add_jump_table(file, insn_t1, NULL);
2122
2123 return ret;
2124 }
2125
2126 /*
2127 * For some switch statements, gcc generates a jump table in the .rodata
2128 * section which contains a list of addresses within the function to jump to.
2129 * This finds these jump tables and adds them to the insn->alts lists.
2130 */
add_jump_table_alts(struct objtool_file * file)2131 static int add_jump_table_alts(struct objtool_file *file)
2132 {
2133 struct symbol *func;
2134 int ret;
2135
2136 if (!file->rodata)
2137 return 0;
2138
2139 for_each_sym(file, func) {
2140 if (func->type != STT_FUNC)
2141 continue;
2142
2143 mark_func_jump_tables(file, func);
2144 ret = add_func_jump_tables(file, func);
2145 if (ret)
2146 return ret;
2147 }
2148
2149 return 0;
2150 }
2151
set_func_state(struct cfi_state * state)2152 static void set_func_state(struct cfi_state *state)
2153 {
2154 state->cfa = initial_func_cfi.cfa;
2155 memcpy(&state->regs, &initial_func_cfi.regs,
2156 CFI_NUM_REGS * sizeof(struct cfi_reg));
2157 state->stack_size = initial_func_cfi.cfa.offset;
2158 state->type = UNWIND_HINT_TYPE_CALL;
2159 }
2160
read_unwind_hints(struct objtool_file * file)2161 static int read_unwind_hints(struct objtool_file *file)
2162 {
2163 struct cfi_state cfi = init_cfi;
2164 struct section *sec;
2165 struct unwind_hint *hint;
2166 struct instruction *insn;
2167 struct reloc *reloc;
2168 unsigned long offset;
2169 int i;
2170
2171 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2172 if (!sec)
2173 return 0;
2174
2175 if (!sec->rsec) {
2176 WARN("missing .rela.discard.unwind_hints section");
2177 return -1;
2178 }
2179
2180 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2181 WARN("struct unwind_hint size mismatch");
2182 return -1;
2183 }
2184
2185 file->hints = true;
2186
2187 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2188 hint = (struct unwind_hint *)sec->data->d_buf + i;
2189
2190 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2191 if (!reloc) {
2192 WARN("can't find reloc for unwind_hints[%d]", i);
2193 return -1;
2194 }
2195
2196 if (reloc->sym->type == STT_SECTION) {
2197 offset = reloc_addend(reloc);
2198 } else if (reloc->sym->local_label) {
2199 offset = reloc->sym->offset;
2200 } else {
2201 WARN("unexpected relocation symbol type in %s", sec->rsec->name);
2202 return -1;
2203 }
2204
2205 insn = find_insn(file, reloc->sym->sec, offset);
2206 if (!insn) {
2207 WARN("can't find insn for unwind_hints[%d]", i);
2208 return -1;
2209 }
2210
2211 insn->hint = true;
2212
2213 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2214 insn->cfi = &force_undefined_cfi;
2215 continue;
2216 }
2217
2218 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2219 insn->hint = false;
2220 insn->save = true;
2221 continue;
2222 }
2223
2224 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2225 insn->restore = true;
2226 continue;
2227 }
2228
2229 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2230 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2231
2232 if (sym && sym->bind == STB_GLOBAL) {
2233 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2234 WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2235 }
2236 }
2237 }
2238
2239 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2240 insn->cfi = &func_cfi;
2241 continue;
2242 }
2243
2244 if (insn->cfi)
2245 cfi = *(insn->cfi);
2246
2247 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2248 WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2249 return -1;
2250 }
2251
2252 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2253 cfi.type = hint->type;
2254 cfi.signal = hint->signal;
2255
2256 insn->cfi = cfi_hash_find_or_add(&cfi);
2257 }
2258
2259 return 0;
2260 }
2261
read_annotate(struct objtool_file * file,int (* func)(struct objtool_file * file,int type,struct instruction * insn))2262 static int read_annotate(struct objtool_file *file,
2263 int (*func)(struct objtool_file *file, int type, struct instruction *insn))
2264 {
2265 struct section *sec;
2266 struct instruction *insn;
2267 struct reloc *reloc;
2268 uint64_t offset;
2269 int type, ret;
2270
2271 sec = find_section_by_name(file->elf, ".discard.annotate_insn");
2272 if (!sec)
2273 return 0;
2274
2275 if (!sec->rsec)
2276 return 0;
2277
2278 if (sec->sh.sh_entsize != 8) {
2279 static bool warned = false;
2280 if (!warned && opts.verbose) {
2281 WARN("%s: dodgy linker, sh_entsize != 8", sec->name);
2282 warned = true;
2283 }
2284 sec->sh.sh_entsize = 8;
2285 }
2286
2287 for_each_reloc(sec->rsec, reloc) {
2288 type = *(u32 *)(sec->data->d_buf + (reloc_idx(reloc) * sec->sh.sh_entsize) + 4);
2289
2290 offset = reloc->sym->offset + reloc_addend(reloc);
2291 insn = find_insn(file, reloc->sym->sec, offset);
2292
2293 if (!insn) {
2294 WARN("bad .discard.annotate_insn entry: %d of type %d", reloc_idx(reloc), type);
2295 return -1;
2296 }
2297
2298 ret = func(file, type, insn);
2299 if (ret < 0)
2300 return ret;
2301 }
2302
2303 return 0;
2304 }
2305
__annotate_early(struct objtool_file * file,int type,struct instruction * insn)2306 static int __annotate_early(struct objtool_file *file, int type, struct instruction *insn)
2307 {
2308 switch (type) {
2309 case ANNOTYPE_IGNORE_ALTS:
2310 insn->ignore_alts = true;
2311 break;
2312
2313 /*
2314 * Must be before read_unwind_hints() since that needs insn->noendbr.
2315 */
2316 case ANNOTYPE_NOENDBR:
2317 insn->noendbr = 1;
2318 break;
2319
2320 default:
2321 break;
2322 }
2323
2324 return 0;
2325 }
2326
__annotate_ifc(struct objtool_file * file,int type,struct instruction * insn)2327 static int __annotate_ifc(struct objtool_file *file, int type, struct instruction *insn)
2328 {
2329 unsigned long dest_off;
2330
2331 if (type != ANNOTYPE_INTRA_FUNCTION_CALL)
2332 return 0;
2333
2334 if (insn->type != INSN_CALL) {
2335 WARN_INSN(insn, "intra_function_call not a direct call");
2336 return -1;
2337 }
2338
2339 /*
2340 * Treat intra-function CALLs as JMPs, but with a stack_op.
2341 * See add_call_destinations(), which strips stack_ops from
2342 * normal CALLs.
2343 */
2344 insn->type = INSN_JUMP_UNCONDITIONAL;
2345
2346 dest_off = arch_jump_destination(insn);
2347 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2348 if (!insn->jump_dest) {
2349 WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2350 insn->sec->name, dest_off);
2351 return -1;
2352 }
2353
2354 return 0;
2355 }
2356
__annotate_late(struct objtool_file * file,int type,struct instruction * insn)2357 static int __annotate_late(struct objtool_file *file, int type, struct instruction *insn)
2358 {
2359 switch (type) {
2360 case ANNOTYPE_NOENDBR:
2361 /* early */
2362 break;
2363
2364 case ANNOTYPE_RETPOLINE_SAFE:
2365 if (insn->type != INSN_JUMP_DYNAMIC &&
2366 insn->type != INSN_CALL_DYNAMIC &&
2367 insn->type != INSN_RETURN &&
2368 insn->type != INSN_NOP) {
2369 WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2370 return -1;
2371 }
2372
2373 insn->retpoline_safe = true;
2374 break;
2375
2376 case ANNOTYPE_INSTR_BEGIN:
2377 insn->instr++;
2378 break;
2379
2380 case ANNOTYPE_INSTR_END:
2381 insn->instr--;
2382 break;
2383
2384 case ANNOTYPE_UNRET_BEGIN:
2385 insn->unret = 1;
2386 break;
2387
2388 case ANNOTYPE_IGNORE_ALTS:
2389 /* early */
2390 break;
2391
2392 case ANNOTYPE_INTRA_FUNCTION_CALL:
2393 /* ifc */
2394 break;
2395
2396 case ANNOTYPE_REACHABLE:
2397 insn->dead_end = false;
2398 break;
2399
2400 default:
2401 WARN_INSN(insn, "Unknown annotation type: %d", type);
2402 break;
2403 }
2404
2405 return 0;
2406 }
2407
2408 /*
2409 * Return true if name matches an instrumentation function, where calls to that
2410 * function from noinstr code can safely be removed, but compilers won't do so.
2411 */
is_profiling_func(const char * name)2412 static bool is_profiling_func(const char *name)
2413 {
2414 /*
2415 * Many compilers cannot disable KCOV with a function attribute.
2416 */
2417 if (!strncmp(name, "__sanitizer_cov_", 16))
2418 return true;
2419
2420 /*
2421 * Some compilers currently do not remove __tsan_func_entry/exit nor
2422 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2423 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2424 * minimum Clang version is 14.0, this can be removed.
2425 */
2426 if (!strncmp(name, "__tsan_func_", 12) ||
2427 !strcmp(name, "__tsan_atomic_signal_fence"))
2428 return true;
2429
2430 return false;
2431 }
2432
classify_symbols(struct objtool_file * file)2433 static int classify_symbols(struct objtool_file *file)
2434 {
2435 struct symbol *func;
2436
2437 for_each_sym(file, func) {
2438 if (func->type == STT_NOTYPE && strstarts(func->name, ".L"))
2439 func->local_label = true;
2440
2441 if (func->bind != STB_GLOBAL)
2442 continue;
2443
2444 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2445 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2446 func->static_call_tramp = true;
2447
2448 if (arch_is_retpoline(func))
2449 func->retpoline_thunk = true;
2450
2451 if (arch_is_rethunk(func))
2452 func->return_thunk = true;
2453
2454 if (arch_is_embedded_insn(func))
2455 func->embedded_insn = true;
2456
2457 if (arch_ftrace_match(func->name))
2458 func->fentry = true;
2459
2460 if (is_profiling_func(func->name))
2461 func->profiling_func = true;
2462 }
2463
2464 return 0;
2465 }
2466
mark_rodata(struct objtool_file * file)2467 static void mark_rodata(struct objtool_file *file)
2468 {
2469 struct section *sec;
2470 bool found = false;
2471
2472 /*
2473 * Search for the following rodata sections, each of which can
2474 * potentially contain jump tables:
2475 *
2476 * - .rodata: can contain GCC switch tables
2477 * - .rodata.<func>: same, if -fdata-sections is being used
2478 * - .data.rel.ro.c_jump_table: contains C annotated jump tables
2479 *
2480 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2481 */
2482 for_each_sec(file, sec) {
2483 if ((!strncmp(sec->name, ".rodata", 7) &&
2484 !strstr(sec->name, ".str1.")) ||
2485 !strncmp(sec->name, ".data.rel.ro", 12)) {
2486 sec->rodata = true;
2487 found = true;
2488 }
2489 }
2490
2491 file->rodata = found;
2492 }
2493
decode_sections(struct objtool_file * file)2494 static int decode_sections(struct objtool_file *file)
2495 {
2496 int ret;
2497
2498 mark_rodata(file);
2499
2500 ret = init_pv_ops(file);
2501 if (ret)
2502 return ret;
2503
2504 /*
2505 * Must be before add_{jump_call}_destination.
2506 */
2507 ret = classify_symbols(file);
2508 if (ret)
2509 return ret;
2510
2511 ret = decode_instructions(file);
2512 if (ret)
2513 return ret;
2514
2515 add_ignores(file);
2516 add_uaccess_safe(file);
2517
2518 ret = read_annotate(file, __annotate_early);
2519 if (ret)
2520 return ret;
2521
2522 /*
2523 * Must be before add_jump_destinations(), which depends on 'func'
2524 * being set for alternatives, to enable proper sibling call detection.
2525 */
2526 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2527 ret = add_special_section_alts(file);
2528 if (ret)
2529 return ret;
2530 }
2531
2532 ret = add_jump_destinations(file);
2533 if (ret)
2534 return ret;
2535
2536 /*
2537 * Must be before add_call_destination(); it changes INSN_CALL to
2538 * INSN_JUMP.
2539 */
2540 ret = read_annotate(file, __annotate_ifc);
2541 if (ret)
2542 return ret;
2543
2544 ret = add_call_destinations(file);
2545 if (ret)
2546 return ret;
2547
2548 ret = add_jump_table_alts(file);
2549 if (ret)
2550 return ret;
2551
2552 ret = read_unwind_hints(file);
2553 if (ret)
2554 return ret;
2555
2556 /*
2557 * Must be after add_call_destinations() such that it can override
2558 * dead_end_function() marks.
2559 */
2560 ret = read_annotate(file, __annotate_late);
2561 if (ret)
2562 return ret;
2563
2564 return 0;
2565 }
2566
is_special_call(struct instruction * insn)2567 static bool is_special_call(struct instruction *insn)
2568 {
2569 if (insn->type == INSN_CALL) {
2570 struct symbol *dest = insn_call_dest(insn);
2571
2572 if (!dest)
2573 return false;
2574
2575 if (dest->fentry || dest->embedded_insn)
2576 return true;
2577 }
2578
2579 return false;
2580 }
2581
has_modified_stack_frame(struct instruction * insn,struct insn_state * state)2582 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2583 {
2584 struct cfi_state *cfi = &state->cfi;
2585 int i;
2586
2587 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2588 return true;
2589
2590 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2591 return true;
2592
2593 if (cfi->stack_size != initial_func_cfi.cfa.offset)
2594 return true;
2595
2596 for (i = 0; i < CFI_NUM_REGS; i++) {
2597 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2598 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2599 return true;
2600 }
2601
2602 return false;
2603 }
2604
check_reg_frame_pos(const struct cfi_reg * reg,int expected_offset)2605 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2606 int expected_offset)
2607 {
2608 return reg->base == CFI_CFA &&
2609 reg->offset == expected_offset;
2610 }
2611
has_valid_stack_frame(struct insn_state * state)2612 static bool has_valid_stack_frame(struct insn_state *state)
2613 {
2614 struct cfi_state *cfi = &state->cfi;
2615
2616 if (cfi->cfa.base == CFI_BP &&
2617 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2618 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2619 return true;
2620
2621 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2622 return true;
2623
2624 return false;
2625 }
2626
update_cfi_state_regs(struct instruction * insn,struct cfi_state * cfi,struct stack_op * op)2627 static int update_cfi_state_regs(struct instruction *insn,
2628 struct cfi_state *cfi,
2629 struct stack_op *op)
2630 {
2631 struct cfi_reg *cfa = &cfi->cfa;
2632
2633 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2634 return 0;
2635
2636 /* push */
2637 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2638 cfa->offset += 8;
2639
2640 /* pop */
2641 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2642 cfa->offset -= 8;
2643
2644 /* add immediate to sp */
2645 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2646 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2647 cfa->offset -= op->src.offset;
2648
2649 return 0;
2650 }
2651
save_reg(struct cfi_state * cfi,unsigned char reg,int base,int offset)2652 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2653 {
2654 if (arch_callee_saved_reg(reg) &&
2655 cfi->regs[reg].base == CFI_UNDEFINED) {
2656 cfi->regs[reg].base = base;
2657 cfi->regs[reg].offset = offset;
2658 }
2659 }
2660
restore_reg(struct cfi_state * cfi,unsigned char reg)2661 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2662 {
2663 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2664 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2665 }
2666
2667 /*
2668 * A note about DRAP stack alignment:
2669 *
2670 * GCC has the concept of a DRAP register, which is used to help keep track of
2671 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2672 * register. The typical DRAP pattern is:
2673 *
2674 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2675 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2676 * 41 ff 72 f8 pushq -0x8(%r10)
2677 * 55 push %rbp
2678 * 48 89 e5 mov %rsp,%rbp
2679 * (more pushes)
2680 * 41 52 push %r10
2681 * ...
2682 * 41 5a pop %r10
2683 * (more pops)
2684 * 5d pop %rbp
2685 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2686 * c3 retq
2687 *
2688 * There are some variations in the epilogues, like:
2689 *
2690 * 5b pop %rbx
2691 * 41 5a pop %r10
2692 * 41 5c pop %r12
2693 * 41 5d pop %r13
2694 * 41 5e pop %r14
2695 * c9 leaveq
2696 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2697 * c3 retq
2698 *
2699 * and:
2700 *
2701 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2702 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2703 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2704 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2705 * c9 leaveq
2706 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2707 * c3 retq
2708 *
2709 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2710 * restored beforehand:
2711 *
2712 * 41 55 push %r13
2713 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2714 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2715 * ...
2716 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2717 * 41 5d pop %r13
2718 * c3 retq
2719 */
update_cfi_state(struct instruction * insn,struct instruction * next_insn,struct cfi_state * cfi,struct stack_op * op)2720 static int update_cfi_state(struct instruction *insn,
2721 struct instruction *next_insn,
2722 struct cfi_state *cfi, struct stack_op *op)
2723 {
2724 struct cfi_reg *cfa = &cfi->cfa;
2725 struct cfi_reg *regs = cfi->regs;
2726
2727 /* ignore UNWIND_HINT_UNDEFINED regions */
2728 if (cfi->force_undefined)
2729 return 0;
2730
2731 /* stack operations don't make sense with an undefined CFA */
2732 if (cfa->base == CFI_UNDEFINED) {
2733 if (insn_func(insn)) {
2734 WARN_INSN(insn, "undefined stack state");
2735 return -1;
2736 }
2737 return 0;
2738 }
2739
2740 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2741 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2742 return update_cfi_state_regs(insn, cfi, op);
2743
2744 switch (op->dest.type) {
2745
2746 case OP_DEST_REG:
2747 switch (op->src.type) {
2748
2749 case OP_SRC_REG:
2750 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2751 cfa->base == CFI_SP &&
2752 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) {
2753
2754 /* mov %rsp, %rbp */
2755 cfa->base = op->dest.reg;
2756 cfi->bp_scratch = false;
2757 }
2758
2759 else if (op->src.reg == CFI_SP &&
2760 op->dest.reg == CFI_BP && cfi->drap) {
2761
2762 /* drap: mov %rsp, %rbp */
2763 regs[CFI_BP].base = CFI_BP;
2764 regs[CFI_BP].offset = -cfi->stack_size;
2765 cfi->bp_scratch = false;
2766 }
2767
2768 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2769
2770 /*
2771 * mov %rsp, %reg
2772 *
2773 * This is needed for the rare case where GCC
2774 * does:
2775 *
2776 * mov %rsp, %rax
2777 * ...
2778 * mov %rax, %rsp
2779 */
2780 cfi->vals[op->dest.reg].base = CFI_CFA;
2781 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2782 }
2783
2784 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2785 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2786
2787 /*
2788 * mov %rbp, %rsp
2789 *
2790 * Restore the original stack pointer (Clang).
2791 */
2792 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2793 }
2794
2795 else if (op->dest.reg == cfa->base) {
2796
2797 /* mov %reg, %rsp */
2798 if (cfa->base == CFI_SP &&
2799 cfi->vals[op->src.reg].base == CFI_CFA) {
2800
2801 /*
2802 * This is needed for the rare case
2803 * where GCC does something dumb like:
2804 *
2805 * lea 0x8(%rsp), %rcx
2806 * ...
2807 * mov %rcx, %rsp
2808 */
2809 cfa->offset = -cfi->vals[op->src.reg].offset;
2810 cfi->stack_size = cfa->offset;
2811
2812 } else if (cfa->base == CFI_SP &&
2813 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2814 cfi->vals[op->src.reg].offset == cfa->offset) {
2815
2816 /*
2817 * Stack swizzle:
2818 *
2819 * 1: mov %rsp, (%[tos])
2820 * 2: mov %[tos], %rsp
2821 * ...
2822 * 3: pop %rsp
2823 *
2824 * Where:
2825 *
2826 * 1 - places a pointer to the previous
2827 * stack at the Top-of-Stack of the
2828 * new stack.
2829 *
2830 * 2 - switches to the new stack.
2831 *
2832 * 3 - pops the Top-of-Stack to restore
2833 * the original stack.
2834 *
2835 * Note: we set base to SP_INDIRECT
2836 * here and preserve offset. Therefore
2837 * when the unwinder reaches ToS it
2838 * will dereference SP and then add the
2839 * offset to find the next frame, IOW:
2840 * (%rsp) + offset.
2841 */
2842 cfa->base = CFI_SP_INDIRECT;
2843
2844 } else {
2845 cfa->base = CFI_UNDEFINED;
2846 cfa->offset = 0;
2847 }
2848 }
2849
2850 else if (op->dest.reg == CFI_SP &&
2851 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2852 cfi->vals[op->src.reg].offset == cfa->offset) {
2853
2854 /*
2855 * The same stack swizzle case 2) as above. But
2856 * because we can't change cfa->base, case 3)
2857 * will become a regular POP. Pretend we're a
2858 * PUSH so things don't go unbalanced.
2859 */
2860 cfi->stack_size += 8;
2861 }
2862
2863
2864 break;
2865
2866 case OP_SRC_ADD:
2867 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2868
2869 /* add imm, %rsp */
2870 cfi->stack_size -= op->src.offset;
2871 if (cfa->base == CFI_SP)
2872 cfa->offset -= op->src.offset;
2873 break;
2874 }
2875
2876 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP &&
2877 insn->sym->frame_pointer) {
2878 /* addi.d fp,sp,imm on LoongArch */
2879 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) {
2880 cfa->base = CFI_BP;
2881 cfa->offset = 0;
2882 }
2883 break;
2884 }
2885
2886 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2887 /* addi.d sp,fp,imm on LoongArch */
2888 if (cfa->base == CFI_BP && cfa->offset == 0) {
2889 if (insn->sym->frame_pointer) {
2890 cfa->base = CFI_SP;
2891 cfa->offset = -op->src.offset;
2892 }
2893 } else {
2894 /* lea disp(%rbp), %rsp */
2895 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2896 }
2897 break;
2898 }
2899
2900 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2901
2902 /* drap: lea disp(%rsp), %drap */
2903 cfi->drap_reg = op->dest.reg;
2904
2905 /*
2906 * lea disp(%rsp), %reg
2907 *
2908 * This is needed for the rare case where GCC
2909 * does something dumb like:
2910 *
2911 * lea 0x8(%rsp), %rcx
2912 * ...
2913 * mov %rcx, %rsp
2914 */
2915 cfi->vals[op->dest.reg].base = CFI_CFA;
2916 cfi->vals[op->dest.reg].offset = \
2917 -cfi->stack_size + op->src.offset;
2918
2919 break;
2920 }
2921
2922 if (cfi->drap && op->dest.reg == CFI_SP &&
2923 op->src.reg == cfi->drap_reg) {
2924
2925 /* drap: lea disp(%drap), %rsp */
2926 cfa->base = CFI_SP;
2927 cfa->offset = cfi->stack_size = -op->src.offset;
2928 cfi->drap_reg = CFI_UNDEFINED;
2929 cfi->drap = false;
2930 break;
2931 }
2932
2933 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
2934 WARN_INSN(insn, "unsupported stack register modification");
2935 return -1;
2936 }
2937
2938 break;
2939
2940 case OP_SRC_AND:
2941 if (op->dest.reg != CFI_SP ||
2942 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2943 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
2944 WARN_INSN(insn, "unsupported stack pointer realignment");
2945 return -1;
2946 }
2947
2948 if (cfi->drap_reg != CFI_UNDEFINED) {
2949 /* drap: and imm, %rsp */
2950 cfa->base = cfi->drap_reg;
2951 cfa->offset = cfi->stack_size = 0;
2952 cfi->drap = true;
2953 }
2954
2955 /*
2956 * Older versions of GCC (4.8ish) realign the stack
2957 * without DRAP, with a frame pointer.
2958 */
2959
2960 break;
2961
2962 case OP_SRC_POP:
2963 case OP_SRC_POPF:
2964 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2965
2966 /* pop %rsp; # restore from a stack swizzle */
2967 cfa->base = CFI_SP;
2968 break;
2969 }
2970
2971 if (!cfi->drap && op->dest.reg == cfa->base) {
2972
2973 /* pop %rbp */
2974 cfa->base = CFI_SP;
2975 }
2976
2977 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2978 op->dest.reg == cfi->drap_reg &&
2979 cfi->drap_offset == -cfi->stack_size) {
2980
2981 /* drap: pop %drap */
2982 cfa->base = cfi->drap_reg;
2983 cfa->offset = 0;
2984 cfi->drap_offset = -1;
2985
2986 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
2987
2988 /* pop %reg */
2989 restore_reg(cfi, op->dest.reg);
2990 }
2991
2992 cfi->stack_size -= 8;
2993 if (cfa->base == CFI_SP)
2994 cfa->offset -= 8;
2995
2996 break;
2997
2998 case OP_SRC_REG_INDIRECT:
2999 if (!cfi->drap && op->dest.reg == cfa->base &&
3000 op->dest.reg == CFI_BP) {
3001
3002 /* mov disp(%rsp), %rbp */
3003 cfa->base = CFI_SP;
3004 cfa->offset = cfi->stack_size;
3005 }
3006
3007 if (cfi->drap && op->src.reg == CFI_BP &&
3008 op->src.offset == cfi->drap_offset) {
3009
3010 /* drap: mov disp(%rbp), %drap */
3011 cfa->base = cfi->drap_reg;
3012 cfa->offset = 0;
3013 cfi->drap_offset = -1;
3014 }
3015
3016 if (cfi->drap && op->src.reg == CFI_BP &&
3017 op->src.offset == regs[op->dest.reg].offset) {
3018
3019 /* drap: mov disp(%rbp), %reg */
3020 restore_reg(cfi, op->dest.reg);
3021
3022 } else if (op->src.reg == cfa->base &&
3023 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3024
3025 /* mov disp(%rbp), %reg */
3026 /* mov disp(%rsp), %reg */
3027 restore_reg(cfi, op->dest.reg);
3028
3029 } else if (op->src.reg == CFI_SP &&
3030 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3031
3032 /* mov disp(%rsp), %reg */
3033 restore_reg(cfi, op->dest.reg);
3034 }
3035
3036 break;
3037
3038 default:
3039 WARN_INSN(insn, "unknown stack-related instruction");
3040 return -1;
3041 }
3042
3043 break;
3044
3045 case OP_DEST_PUSH:
3046 case OP_DEST_PUSHF:
3047 cfi->stack_size += 8;
3048 if (cfa->base == CFI_SP)
3049 cfa->offset += 8;
3050
3051 if (op->src.type != OP_SRC_REG)
3052 break;
3053
3054 if (cfi->drap) {
3055 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3056
3057 /* drap: push %drap */
3058 cfa->base = CFI_BP_INDIRECT;
3059 cfa->offset = -cfi->stack_size;
3060
3061 /* save drap so we know when to restore it */
3062 cfi->drap_offset = -cfi->stack_size;
3063
3064 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3065
3066 /* drap: push %rbp */
3067 cfi->stack_size = 0;
3068
3069 } else {
3070
3071 /* drap: push %reg */
3072 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3073 }
3074
3075 } else {
3076
3077 /* push %reg */
3078 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3079 }
3080
3081 /* detect when asm code uses rbp as a scratch register */
3082 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3083 cfa->base != CFI_BP)
3084 cfi->bp_scratch = true;
3085 break;
3086
3087 case OP_DEST_REG_INDIRECT:
3088
3089 if (cfi->drap) {
3090 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3091
3092 /* drap: mov %drap, disp(%rbp) */
3093 cfa->base = CFI_BP_INDIRECT;
3094 cfa->offset = op->dest.offset;
3095
3096 /* save drap offset so we know when to restore it */
3097 cfi->drap_offset = op->dest.offset;
3098 } else {
3099
3100 /* drap: mov reg, disp(%rbp) */
3101 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3102 }
3103
3104 } else if (op->dest.reg == cfa->base) {
3105
3106 /* mov reg, disp(%rbp) */
3107 /* mov reg, disp(%rsp) */
3108 save_reg(cfi, op->src.reg, CFI_CFA,
3109 op->dest.offset - cfi->cfa.offset);
3110
3111 } else if (op->dest.reg == CFI_SP) {
3112
3113 /* mov reg, disp(%rsp) */
3114 save_reg(cfi, op->src.reg, CFI_CFA,
3115 op->dest.offset - cfi->stack_size);
3116
3117 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3118
3119 /* mov %rsp, (%reg); # setup a stack swizzle. */
3120 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3121 cfi->vals[op->dest.reg].offset = cfa->offset;
3122 }
3123
3124 break;
3125
3126 case OP_DEST_MEM:
3127 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3128 WARN_INSN(insn, "unknown stack-related memory operation");
3129 return -1;
3130 }
3131
3132 /* pop mem */
3133 cfi->stack_size -= 8;
3134 if (cfa->base == CFI_SP)
3135 cfa->offset -= 8;
3136
3137 break;
3138
3139 default:
3140 WARN_INSN(insn, "unknown stack-related instruction");
3141 return -1;
3142 }
3143
3144 return 0;
3145 }
3146
3147 /*
3148 * The stack layouts of alternatives instructions can sometimes diverge when
3149 * they have stack modifications. That's fine as long as the potential stack
3150 * layouts don't conflict at any given potential instruction boundary.
3151 *
3152 * Flatten the CFIs of the different alternative code streams (both original
3153 * and replacement) into a single shared CFI array which can be used to detect
3154 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3155 */
propagate_alt_cfi(struct objtool_file * file,struct instruction * insn)3156 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3157 {
3158 struct cfi_state **alt_cfi;
3159 int group_off;
3160
3161 if (!insn->alt_group)
3162 return 0;
3163
3164 if (!insn->cfi) {
3165 WARN("CFI missing");
3166 return -1;
3167 }
3168
3169 alt_cfi = insn->alt_group->cfi;
3170 group_off = insn->offset - insn->alt_group->first_insn->offset;
3171
3172 if (!alt_cfi[group_off]) {
3173 alt_cfi[group_off] = insn->cfi;
3174 } else {
3175 if (cficmp(alt_cfi[group_off], insn->cfi)) {
3176 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3177 struct instruction *orig = orig_group->first_insn;
3178 char *where = offstr(insn->sec, insn->offset);
3179 WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3180 free(where);
3181 return -1;
3182 }
3183 }
3184
3185 return 0;
3186 }
3187
handle_insn_ops(struct instruction * insn,struct instruction * next_insn,struct insn_state * state)3188 static int handle_insn_ops(struct instruction *insn,
3189 struct instruction *next_insn,
3190 struct insn_state *state)
3191 {
3192 struct stack_op *op;
3193
3194 for (op = insn->stack_ops; op; op = op->next) {
3195
3196 if (update_cfi_state(insn, next_insn, &state->cfi, op))
3197 return 1;
3198
3199 if (!insn->alt_group)
3200 continue;
3201
3202 if (op->dest.type == OP_DEST_PUSHF) {
3203 if (!state->uaccess_stack) {
3204 state->uaccess_stack = 1;
3205 } else if (state->uaccess_stack >> 31) {
3206 WARN_INSN(insn, "PUSHF stack exhausted");
3207 return 1;
3208 }
3209 state->uaccess_stack <<= 1;
3210 state->uaccess_stack |= state->uaccess;
3211 }
3212
3213 if (op->src.type == OP_SRC_POPF) {
3214 if (state->uaccess_stack) {
3215 state->uaccess = state->uaccess_stack & 1;
3216 state->uaccess_stack >>= 1;
3217 if (state->uaccess_stack == 1)
3218 state->uaccess_stack = 0;
3219 }
3220 }
3221 }
3222
3223 return 0;
3224 }
3225
insn_cfi_match(struct instruction * insn,struct cfi_state * cfi2)3226 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3227 {
3228 struct cfi_state *cfi1 = insn->cfi;
3229 int i;
3230
3231 if (!cfi1) {
3232 WARN("CFI missing");
3233 return false;
3234 }
3235
3236 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3237
3238 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3239 cfi1->cfa.base, cfi1->cfa.offset,
3240 cfi2->cfa.base, cfi2->cfa.offset);
3241
3242 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3243 for (i = 0; i < CFI_NUM_REGS; i++) {
3244 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3245 sizeof(struct cfi_reg)))
3246 continue;
3247
3248 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3249 i, cfi1->regs[i].base, cfi1->regs[i].offset,
3250 i, cfi2->regs[i].base, cfi2->regs[i].offset);
3251 break;
3252 }
3253
3254 } else if (cfi1->type != cfi2->type) {
3255
3256 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3257 cfi1->type, cfi2->type);
3258
3259 } else if (cfi1->drap != cfi2->drap ||
3260 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3261 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3262
3263 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3264 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3265 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3266
3267 } else
3268 return true;
3269
3270 return false;
3271 }
3272
func_uaccess_safe(struct symbol * func)3273 static inline bool func_uaccess_safe(struct symbol *func)
3274 {
3275 if (func)
3276 return func->uaccess_safe;
3277
3278 return false;
3279 }
3280
call_dest_name(struct instruction * insn)3281 static inline const char *call_dest_name(struct instruction *insn)
3282 {
3283 static char pvname[19];
3284 struct reloc *reloc;
3285 int idx;
3286
3287 if (insn_call_dest(insn))
3288 return insn_call_dest(insn)->name;
3289
3290 reloc = insn_reloc(NULL, insn);
3291 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3292 idx = (reloc_addend(reloc) / sizeof(void *));
3293 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3294 return pvname;
3295 }
3296
3297 return "{dynamic}";
3298 }
3299
pv_call_dest(struct objtool_file * file,struct instruction * insn)3300 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3301 {
3302 struct symbol *target;
3303 struct reloc *reloc;
3304 int idx;
3305
3306 reloc = insn_reloc(file, insn);
3307 if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3308 return false;
3309
3310 idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3311
3312 if (file->pv_ops[idx].clean)
3313 return true;
3314
3315 file->pv_ops[idx].clean = true;
3316
3317 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3318 if (!target->sec->noinstr) {
3319 WARN("pv_ops[%d]: %s", idx, target->name);
3320 file->pv_ops[idx].clean = false;
3321 }
3322 }
3323
3324 return file->pv_ops[idx].clean;
3325 }
3326
noinstr_call_dest(struct objtool_file * file,struct instruction * insn,struct symbol * func)3327 static inline bool noinstr_call_dest(struct objtool_file *file,
3328 struct instruction *insn,
3329 struct symbol *func)
3330 {
3331 /*
3332 * We can't deal with indirect function calls at present;
3333 * assume they're instrumented.
3334 */
3335 if (!func) {
3336 if (file->pv_ops)
3337 return pv_call_dest(file, insn);
3338
3339 return false;
3340 }
3341
3342 /*
3343 * If the symbol is from a noinstr section; we good.
3344 */
3345 if (func->sec->noinstr)
3346 return true;
3347
3348 /*
3349 * If the symbol is a static_call trampoline, we can't tell.
3350 */
3351 if (func->static_call_tramp)
3352 return true;
3353
3354 /*
3355 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3356 * something 'BAD' happened. At the risk of taking the machine down,
3357 * let them proceed to get the message out.
3358 */
3359 if (!strncmp(func->name, "__ubsan_handle_", 15))
3360 return true;
3361
3362 return false;
3363 }
3364
validate_call(struct objtool_file * file,struct instruction * insn,struct insn_state * state)3365 static int validate_call(struct objtool_file *file,
3366 struct instruction *insn,
3367 struct insn_state *state)
3368 {
3369 if (state->noinstr && state->instr <= 0 &&
3370 !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3371 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3372 return 1;
3373 }
3374
3375 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3376 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3377 return 1;
3378 }
3379
3380 if (state->df) {
3381 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3382 return 1;
3383 }
3384
3385 return 0;
3386 }
3387
validate_sibling_call(struct objtool_file * file,struct instruction * insn,struct insn_state * state)3388 static int validate_sibling_call(struct objtool_file *file,
3389 struct instruction *insn,
3390 struct insn_state *state)
3391 {
3392 if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3393 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3394 return 1;
3395 }
3396
3397 return validate_call(file, insn, state);
3398 }
3399
validate_return(struct symbol * func,struct instruction * insn,struct insn_state * state)3400 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3401 {
3402 if (state->noinstr && state->instr > 0) {
3403 WARN_INSN(insn, "return with instrumentation enabled");
3404 return 1;
3405 }
3406
3407 if (state->uaccess && !func_uaccess_safe(func)) {
3408 WARN_INSN(insn, "return with UACCESS enabled");
3409 return 1;
3410 }
3411
3412 if (!state->uaccess && func_uaccess_safe(func)) {
3413 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3414 return 1;
3415 }
3416
3417 if (state->df) {
3418 WARN_INSN(insn, "return with DF set");
3419 return 1;
3420 }
3421
3422 if (func && has_modified_stack_frame(insn, state)) {
3423 WARN_INSN(insn, "return with modified stack frame");
3424 return 1;
3425 }
3426
3427 if (state->cfi.bp_scratch) {
3428 WARN_INSN(insn, "BP used as a scratch register");
3429 return 1;
3430 }
3431
3432 return 0;
3433 }
3434
next_insn_to_validate(struct objtool_file * file,struct instruction * insn)3435 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3436 struct instruction *insn)
3437 {
3438 struct alt_group *alt_group = insn->alt_group;
3439
3440 /*
3441 * Simulate the fact that alternatives are patched in-place. When the
3442 * end of a replacement alt_group is reached, redirect objtool flow to
3443 * the end of the original alt_group.
3444 *
3445 * insn->alts->insn -> alt_group->first_insn
3446 * ...
3447 * alt_group->last_insn
3448 * [alt_group->nop] -> next(orig_group->last_insn)
3449 */
3450 if (alt_group) {
3451 if (alt_group->nop) {
3452 /* ->nop implies ->orig_group */
3453 if (insn == alt_group->last_insn)
3454 return alt_group->nop;
3455 if (insn == alt_group->nop)
3456 goto next_orig;
3457 }
3458 if (insn == alt_group->last_insn && alt_group->orig_group)
3459 goto next_orig;
3460 }
3461
3462 return next_insn_same_sec(file, insn);
3463
3464 next_orig:
3465 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3466 }
3467
3468 /*
3469 * Follow the branch starting at the given instruction, and recursively follow
3470 * any other branches (jumps). Meanwhile, track the frame pointer state at
3471 * each instruction and validate all the rules described in
3472 * tools/objtool/Documentation/objtool.txt.
3473 */
validate_branch(struct objtool_file * file,struct symbol * func,struct instruction * insn,struct insn_state state)3474 static int validate_branch(struct objtool_file *file, struct symbol *func,
3475 struct instruction *insn, struct insn_state state)
3476 {
3477 struct alternative *alt;
3478 struct instruction *next_insn, *prev_insn = NULL;
3479 struct section *sec;
3480 u8 visited;
3481 int ret;
3482
3483 sec = insn->sec;
3484
3485 while (1) {
3486 next_insn = next_insn_to_validate(file, insn);
3487
3488 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3489 /* Ignore KCFI type preambles, which always fall through */
3490 if (!strncmp(func->name, "__cfi_", 6) ||
3491 !strncmp(func->name, "__pfx_", 6))
3492 return 0;
3493
3494 WARN("%s() falls through to next function %s()",
3495 func->name, insn_func(insn)->name);
3496 return 1;
3497 }
3498
3499 if (func && insn->ignore) {
3500 WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3501 return 1;
3502 }
3503
3504 visited = VISITED_BRANCH << state.uaccess;
3505 if (insn->visited & VISITED_BRANCH_MASK) {
3506 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3507 return 1;
3508
3509 if (insn->visited & visited)
3510 return 0;
3511 } else {
3512 nr_insns_visited++;
3513 }
3514
3515 if (state.noinstr)
3516 state.instr += insn->instr;
3517
3518 if (insn->hint) {
3519 if (insn->restore) {
3520 struct instruction *save_insn, *i;
3521
3522 i = insn;
3523 save_insn = NULL;
3524
3525 sym_for_each_insn_continue_reverse(file, func, i) {
3526 if (i->save) {
3527 save_insn = i;
3528 break;
3529 }
3530 }
3531
3532 if (!save_insn) {
3533 WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3534 return 1;
3535 }
3536
3537 if (!save_insn->visited) {
3538 /*
3539 * If the restore hint insn is at the
3540 * beginning of a basic block and was
3541 * branched to from elsewhere, and the
3542 * save insn hasn't been visited yet,
3543 * defer following this branch for now.
3544 * It will be seen later via the
3545 * straight-line path.
3546 */
3547 if (!prev_insn)
3548 return 0;
3549
3550 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3551 return 1;
3552 }
3553
3554 insn->cfi = save_insn->cfi;
3555 nr_cfi_reused++;
3556 }
3557
3558 state.cfi = *insn->cfi;
3559 } else {
3560 /* XXX track if we actually changed state.cfi */
3561
3562 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3563 insn->cfi = prev_insn->cfi;
3564 nr_cfi_reused++;
3565 } else {
3566 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3567 }
3568 }
3569
3570 insn->visited |= visited;
3571
3572 if (propagate_alt_cfi(file, insn))
3573 return 1;
3574
3575 if (!insn->ignore_alts && insn->alts) {
3576 bool skip_orig = false;
3577
3578 for (alt = insn->alts; alt; alt = alt->next) {
3579 if (alt->skip_orig)
3580 skip_orig = true;
3581
3582 ret = validate_branch(file, func, alt->insn, state);
3583 if (ret) {
3584 BT_INSN(insn, "(alt)");
3585 return ret;
3586 }
3587 }
3588
3589 if (skip_orig)
3590 return 0;
3591 }
3592
3593 if (handle_insn_ops(insn, next_insn, &state))
3594 return 1;
3595
3596 switch (insn->type) {
3597
3598 case INSN_RETURN:
3599 return validate_return(func, insn, &state);
3600
3601 case INSN_CALL:
3602 case INSN_CALL_DYNAMIC:
3603 ret = validate_call(file, insn, &state);
3604 if (ret)
3605 return ret;
3606
3607 if (opts.stackval && func && !is_special_call(insn) &&
3608 !has_valid_stack_frame(&state)) {
3609 WARN_INSN(insn, "call without frame pointer save/setup");
3610 return 1;
3611 }
3612
3613 if (insn->dead_end)
3614 return 0;
3615
3616 break;
3617
3618 case INSN_JUMP_CONDITIONAL:
3619 case INSN_JUMP_UNCONDITIONAL:
3620 if (is_sibling_call(insn)) {
3621 ret = validate_sibling_call(file, insn, &state);
3622 if (ret)
3623 return ret;
3624
3625 } else if (insn->jump_dest) {
3626 ret = validate_branch(file, func,
3627 insn->jump_dest, state);
3628 if (ret) {
3629 BT_INSN(insn, "(branch)");
3630 return ret;
3631 }
3632 }
3633
3634 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3635 return 0;
3636
3637 break;
3638
3639 case INSN_JUMP_DYNAMIC:
3640 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3641 if (is_sibling_call(insn)) {
3642 ret = validate_sibling_call(file, insn, &state);
3643 if (ret)
3644 return ret;
3645 }
3646
3647 if (insn->type == INSN_JUMP_DYNAMIC)
3648 return 0;
3649
3650 break;
3651
3652 case INSN_CONTEXT_SWITCH:
3653 if (func) {
3654 if (!next_insn || !next_insn->hint) {
3655 WARN_INSN(insn, "unsupported instruction in callable function");
3656 return 1;
3657 }
3658 break;
3659 }
3660 return 0;
3661
3662 case INSN_STAC:
3663 if (state.uaccess) {
3664 WARN_INSN(insn, "recursive UACCESS enable");
3665 return 1;
3666 }
3667
3668 state.uaccess = true;
3669 break;
3670
3671 case INSN_CLAC:
3672 if (!state.uaccess && func) {
3673 WARN_INSN(insn, "redundant UACCESS disable");
3674 return 1;
3675 }
3676
3677 if (func_uaccess_safe(func) && !state.uaccess_stack) {
3678 WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3679 return 1;
3680 }
3681
3682 state.uaccess = false;
3683 break;
3684
3685 case INSN_STD:
3686 if (state.df) {
3687 WARN_INSN(insn, "recursive STD");
3688 return 1;
3689 }
3690
3691 state.df = true;
3692 break;
3693
3694 case INSN_CLD:
3695 if (!state.df && func) {
3696 WARN_INSN(insn, "redundant CLD");
3697 return 1;
3698 }
3699
3700 state.df = false;
3701 break;
3702
3703 default:
3704 break;
3705 }
3706
3707 if (insn->dead_end)
3708 return 0;
3709
3710 if (!next_insn) {
3711 if (state.cfi.cfa.base == CFI_UNDEFINED)
3712 return 0;
3713 WARN("%s: unexpected end of section", sec->name);
3714 return 1;
3715 }
3716
3717 prev_insn = insn;
3718 insn = next_insn;
3719 }
3720
3721 return 0;
3722 }
3723
validate_unwind_hint(struct objtool_file * file,struct instruction * insn,struct insn_state * state)3724 static int validate_unwind_hint(struct objtool_file *file,
3725 struct instruction *insn,
3726 struct insn_state *state)
3727 {
3728 if (insn->hint && !insn->visited && !insn->ignore) {
3729 int ret = validate_branch(file, insn_func(insn), insn, *state);
3730 if (ret)
3731 BT_INSN(insn, "<=== (hint)");
3732 return ret;
3733 }
3734
3735 return 0;
3736 }
3737
validate_unwind_hints(struct objtool_file * file,struct section * sec)3738 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3739 {
3740 struct instruction *insn;
3741 struct insn_state state;
3742 int warnings = 0;
3743
3744 if (!file->hints)
3745 return 0;
3746
3747 init_insn_state(file, &state, sec);
3748
3749 if (sec) {
3750 sec_for_each_insn(file, sec, insn)
3751 warnings += validate_unwind_hint(file, insn, &state);
3752 } else {
3753 for_each_insn(file, insn)
3754 warnings += validate_unwind_hint(file, insn, &state);
3755 }
3756
3757 return warnings;
3758 }
3759
3760 /*
3761 * Validate rethunk entry constraint: must untrain RET before the first RET.
3762 *
3763 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3764 * before an actual RET instruction.
3765 */
validate_unret(struct objtool_file * file,struct instruction * insn)3766 static int validate_unret(struct objtool_file *file, struct instruction *insn)
3767 {
3768 struct instruction *next, *dest;
3769 int ret;
3770
3771 for (;;) {
3772 next = next_insn_to_validate(file, insn);
3773
3774 if (insn->visited & VISITED_UNRET)
3775 return 0;
3776
3777 insn->visited |= VISITED_UNRET;
3778
3779 if (!insn->ignore_alts && insn->alts) {
3780 struct alternative *alt;
3781 bool skip_orig = false;
3782
3783 for (alt = insn->alts; alt; alt = alt->next) {
3784 if (alt->skip_orig)
3785 skip_orig = true;
3786
3787 ret = validate_unret(file, alt->insn);
3788 if (ret) {
3789 BT_INSN(insn, "(alt)");
3790 return ret;
3791 }
3792 }
3793
3794 if (skip_orig)
3795 return 0;
3796 }
3797
3798 switch (insn->type) {
3799
3800 case INSN_CALL_DYNAMIC:
3801 case INSN_JUMP_DYNAMIC:
3802 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3803 WARN_INSN(insn, "early indirect call");
3804 return 1;
3805
3806 case INSN_JUMP_UNCONDITIONAL:
3807 case INSN_JUMP_CONDITIONAL:
3808 if (!is_sibling_call(insn)) {
3809 if (!insn->jump_dest) {
3810 WARN_INSN(insn, "unresolved jump target after linking?!?");
3811 return -1;
3812 }
3813 ret = validate_unret(file, insn->jump_dest);
3814 if (ret) {
3815 BT_INSN(insn, "(branch%s)",
3816 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3817 return ret;
3818 }
3819
3820 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3821 return 0;
3822
3823 break;
3824 }
3825
3826 /* fallthrough */
3827 case INSN_CALL:
3828 dest = find_insn(file, insn_call_dest(insn)->sec,
3829 insn_call_dest(insn)->offset);
3830 if (!dest) {
3831 WARN("Unresolved function after linking!?: %s",
3832 insn_call_dest(insn)->name);
3833 return -1;
3834 }
3835
3836 ret = validate_unret(file, dest);
3837 if (ret) {
3838 BT_INSN(insn, "(call)");
3839 return ret;
3840 }
3841 /*
3842 * If a call returns without error, it must have seen UNTRAIN_RET.
3843 * Therefore any non-error return is a success.
3844 */
3845 return 0;
3846
3847 case INSN_RETURN:
3848 WARN_INSN(insn, "RET before UNTRAIN");
3849 return 1;
3850
3851 case INSN_NOP:
3852 if (insn->retpoline_safe)
3853 return 0;
3854 break;
3855
3856 default:
3857 break;
3858 }
3859
3860 if (!next) {
3861 WARN_INSN(insn, "teh end!");
3862 return -1;
3863 }
3864 insn = next;
3865 }
3866
3867 return 0;
3868 }
3869
3870 /*
3871 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3872 * VALIDATE_UNRET_END before RET.
3873 */
validate_unrets(struct objtool_file * file)3874 static int validate_unrets(struct objtool_file *file)
3875 {
3876 struct instruction *insn;
3877 int ret, warnings = 0;
3878
3879 for_each_insn(file, insn) {
3880 if (!insn->unret)
3881 continue;
3882
3883 ret = validate_unret(file, insn);
3884 if (ret < 0) {
3885 WARN_INSN(insn, "Failed UNRET validation");
3886 return ret;
3887 }
3888 warnings += ret;
3889 }
3890
3891 return warnings;
3892 }
3893
validate_retpoline(struct objtool_file * file)3894 static int validate_retpoline(struct objtool_file *file)
3895 {
3896 struct instruction *insn;
3897 int warnings = 0;
3898
3899 for_each_insn(file, insn) {
3900 if (insn->type != INSN_JUMP_DYNAMIC &&
3901 insn->type != INSN_CALL_DYNAMIC &&
3902 insn->type != INSN_RETURN)
3903 continue;
3904
3905 if (insn->retpoline_safe)
3906 continue;
3907
3908 if (insn->sec->init)
3909 continue;
3910
3911 if (insn->type == INSN_RETURN) {
3912 if (opts.rethunk) {
3913 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
3914 } else
3915 continue;
3916 } else {
3917 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
3918 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3919 }
3920
3921 warnings++;
3922 }
3923
3924 return warnings;
3925 }
3926
is_kasan_insn(struct instruction * insn)3927 static bool is_kasan_insn(struct instruction *insn)
3928 {
3929 return (insn->type == INSN_CALL &&
3930 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
3931 }
3932
is_ubsan_insn(struct instruction * insn)3933 static bool is_ubsan_insn(struct instruction *insn)
3934 {
3935 return (insn->type == INSN_CALL &&
3936 !strcmp(insn_call_dest(insn)->name,
3937 "__ubsan_handle_builtin_unreachable"));
3938 }
3939
ignore_unreachable_insn(struct objtool_file * file,struct instruction * insn)3940 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
3941 {
3942 int i;
3943 struct instruction *prev_insn;
3944
3945 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
3946 return true;
3947
3948 /*
3949 * Ignore alternative replacement instructions. This can happen
3950 * when a whitelisted function uses one of the ALTERNATIVE macros.
3951 */
3952 if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
3953 !strcmp(insn->sec->name, ".altinstr_aux"))
3954 return true;
3955
3956 /*
3957 * Whole archive runs might encounter dead code from weak symbols.
3958 * This is where the linker will have dropped the weak symbol in
3959 * favour of a regular symbol, but leaves the code in place.
3960 *
3961 * In this case we'll find a piece of code (whole function) that is not
3962 * covered by a !section symbol. Ignore them.
3963 */
3964 if (opts.link && !insn_func(insn)) {
3965 int size = find_symbol_hole_containing(insn->sec, insn->offset);
3966 unsigned long end = insn->offset + size;
3967
3968 if (!size) /* not a hole */
3969 return false;
3970
3971 if (size < 0) /* hole until the end */
3972 return true;
3973
3974 sec_for_each_insn_continue(file, insn) {
3975 /*
3976 * If we reach a visited instruction at or before the
3977 * end of the hole, ignore the unreachable.
3978 */
3979 if (insn->visited)
3980 return true;
3981
3982 if (insn->offset >= end)
3983 break;
3984
3985 /*
3986 * If this hole jumps to a .cold function, mark it ignore too.
3987 */
3988 if (insn->jump_dest && insn_func(insn->jump_dest) &&
3989 strstr(insn_func(insn->jump_dest)->name, ".cold")) {
3990 struct instruction *dest = insn->jump_dest;
3991 func_for_each_insn(file, insn_func(dest), dest)
3992 dest->ignore = true;
3993 }
3994 }
3995
3996 return false;
3997 }
3998
3999 if (!insn_func(insn))
4000 return false;
4001
4002 if (insn_func(insn)->static_call_tramp)
4003 return true;
4004
4005 /*
4006 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4007 * __builtin_unreachable(). The BUG() macro has an unreachable() after
4008 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4009 * (or occasionally a JMP to UD2).
4010 *
4011 * It may also insert a UD2 after calling a __noreturn function.
4012 */
4013 prev_insn = prev_insn_same_sec(file, insn);
4014 if (prev_insn->dead_end &&
4015 (insn->type == INSN_BUG ||
4016 (insn->type == INSN_JUMP_UNCONDITIONAL &&
4017 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4018 return true;
4019
4020 /*
4021 * Check if this (or a subsequent) instruction is related to
4022 * CONFIG_UBSAN or CONFIG_KASAN.
4023 *
4024 * End the search at 5 instructions to avoid going into the weeds.
4025 */
4026 for (i = 0; i < 5; i++) {
4027
4028 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4029 return true;
4030
4031 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4032 if (insn->jump_dest &&
4033 insn_func(insn->jump_dest) == insn_func(insn)) {
4034 insn = insn->jump_dest;
4035 continue;
4036 }
4037
4038 break;
4039 }
4040
4041 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4042 break;
4043
4044 insn = next_insn_same_sec(file, insn);
4045 }
4046
4047 return false;
4048 }
4049
add_prefix_symbol(struct objtool_file * file,struct symbol * func)4050 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4051 {
4052 struct instruction *insn, *prev;
4053 struct cfi_state *cfi;
4054
4055 insn = find_insn(file, func->sec, func->offset);
4056 if (!insn)
4057 return -1;
4058
4059 for (prev = prev_insn_same_sec(file, insn);
4060 prev;
4061 prev = prev_insn_same_sec(file, prev)) {
4062 u64 offset;
4063
4064 if (prev->type != INSN_NOP)
4065 return -1;
4066
4067 offset = func->offset - prev->offset;
4068
4069 if (offset > opts.prefix)
4070 return -1;
4071
4072 if (offset < opts.prefix)
4073 continue;
4074
4075 elf_create_prefix_symbol(file->elf, func, opts.prefix);
4076 break;
4077 }
4078
4079 if (!prev)
4080 return -1;
4081
4082 if (!insn->cfi) {
4083 /*
4084 * This can happen if stack validation isn't enabled or the
4085 * function is annotated with STACK_FRAME_NON_STANDARD.
4086 */
4087 return 0;
4088 }
4089
4090 /* Propagate insn->cfi to the prefix code */
4091 cfi = cfi_hash_find_or_add(insn->cfi);
4092 for (; prev != insn; prev = next_insn_same_sec(file, prev))
4093 prev->cfi = cfi;
4094
4095 return 0;
4096 }
4097
add_prefix_symbols(struct objtool_file * file)4098 static int add_prefix_symbols(struct objtool_file *file)
4099 {
4100 struct section *sec;
4101 struct symbol *func;
4102
4103 for_each_sec(file, sec) {
4104 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4105 continue;
4106
4107 sec_for_each_sym(sec, func) {
4108 if (func->type != STT_FUNC)
4109 continue;
4110
4111 add_prefix_symbol(file, func);
4112 }
4113 }
4114
4115 return 0;
4116 }
4117
validate_symbol(struct objtool_file * file,struct section * sec,struct symbol * sym,struct insn_state * state)4118 static int validate_symbol(struct objtool_file *file, struct section *sec,
4119 struct symbol *sym, struct insn_state *state)
4120 {
4121 struct instruction *insn;
4122 int ret;
4123
4124 if (!sym->len) {
4125 WARN("%s() is missing an ELF size annotation", sym->name);
4126 return 1;
4127 }
4128
4129 if (sym->pfunc != sym || sym->alias != sym)
4130 return 0;
4131
4132 insn = find_insn(file, sec, sym->offset);
4133 if (!insn || insn->ignore || insn->visited)
4134 return 0;
4135
4136 state->uaccess = sym->uaccess_safe;
4137
4138 ret = validate_branch(file, insn_func(insn), insn, *state);
4139 if (ret)
4140 BT_INSN(insn, "<=== (sym)");
4141 return ret;
4142 }
4143
validate_section(struct objtool_file * file,struct section * sec)4144 static int validate_section(struct objtool_file *file, struct section *sec)
4145 {
4146 struct insn_state state;
4147 struct symbol *func;
4148 int warnings = 0;
4149
4150 sec_for_each_sym(sec, func) {
4151 if (func->type != STT_FUNC)
4152 continue;
4153
4154 init_insn_state(file, &state, sec);
4155 set_func_state(&state.cfi);
4156
4157 warnings += validate_symbol(file, sec, func, &state);
4158 }
4159
4160 return warnings;
4161 }
4162
validate_noinstr_sections(struct objtool_file * file)4163 static int validate_noinstr_sections(struct objtool_file *file)
4164 {
4165 struct section *sec;
4166 int warnings = 0;
4167
4168 sec = find_section_by_name(file->elf, ".noinstr.text");
4169 if (sec) {
4170 warnings += validate_section(file, sec);
4171 warnings += validate_unwind_hints(file, sec);
4172 }
4173
4174 sec = find_section_by_name(file->elf, ".entry.text");
4175 if (sec) {
4176 warnings += validate_section(file, sec);
4177 warnings += validate_unwind_hints(file, sec);
4178 }
4179
4180 sec = find_section_by_name(file->elf, ".cpuidle.text");
4181 if (sec) {
4182 warnings += validate_section(file, sec);
4183 warnings += validate_unwind_hints(file, sec);
4184 }
4185
4186 return warnings;
4187 }
4188
validate_functions(struct objtool_file * file)4189 static int validate_functions(struct objtool_file *file)
4190 {
4191 struct section *sec;
4192 int warnings = 0;
4193
4194 for_each_sec(file, sec) {
4195 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4196 continue;
4197
4198 warnings += validate_section(file, sec);
4199 }
4200
4201 return warnings;
4202 }
4203
mark_endbr_used(struct instruction * insn)4204 static void mark_endbr_used(struct instruction *insn)
4205 {
4206 if (!list_empty(&insn->call_node))
4207 list_del_init(&insn->call_node);
4208 }
4209
noendbr_range(struct objtool_file * file,struct instruction * insn)4210 static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4211 {
4212 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4213 struct instruction *first;
4214
4215 if (!sym)
4216 return false;
4217
4218 first = find_insn(file, sym->sec, sym->offset);
4219 if (!first)
4220 return false;
4221
4222 if (first->type != INSN_ENDBR && !first->noendbr)
4223 return false;
4224
4225 return insn->offset == sym->offset + sym->len;
4226 }
4227
__validate_ibt_insn(struct objtool_file * file,struct instruction * insn,struct instruction * dest)4228 static int __validate_ibt_insn(struct objtool_file *file, struct instruction *insn,
4229 struct instruction *dest)
4230 {
4231 if (dest->type == INSN_ENDBR) {
4232 mark_endbr_used(dest);
4233 return 0;
4234 }
4235
4236 if (insn_func(dest) && insn_func(insn) &&
4237 insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4238 /*
4239 * Anything from->to self is either _THIS_IP_ or
4240 * IRET-to-self.
4241 *
4242 * There is no sane way to annotate _THIS_IP_ since the
4243 * compiler treats the relocation as a constant and is
4244 * happy to fold in offsets, skewing any annotation we
4245 * do, leading to vast amounts of false-positives.
4246 *
4247 * There's also compiler generated _THIS_IP_ through
4248 * KCOV and such which we have no hope of annotating.
4249 *
4250 * As such, blanket accept self-references without
4251 * issue.
4252 */
4253 return 0;
4254 }
4255
4256 /*
4257 * Accept anything ANNOTATE_NOENDBR.
4258 */
4259 if (dest->noendbr)
4260 return 0;
4261
4262 /*
4263 * Accept if this is the instruction after a symbol
4264 * that is (no)endbr -- typical code-range usage.
4265 */
4266 if (noendbr_range(file, dest))
4267 return 0;
4268
4269 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4270 return 1;
4271 }
4272
validate_ibt_insn(struct objtool_file * file,struct instruction * insn)4273 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4274 {
4275 struct instruction *dest;
4276 struct reloc *reloc;
4277 unsigned long off;
4278 int warnings = 0;
4279
4280 /*
4281 * Looking for function pointer load relocations. Ignore
4282 * direct/indirect branches:
4283 */
4284 switch (insn->type) {
4285
4286 case INSN_CALL:
4287 case INSN_CALL_DYNAMIC:
4288 case INSN_JUMP_CONDITIONAL:
4289 case INSN_JUMP_UNCONDITIONAL:
4290 case INSN_JUMP_DYNAMIC:
4291 case INSN_JUMP_DYNAMIC_CONDITIONAL:
4292 case INSN_RETURN:
4293 case INSN_NOP:
4294 return 0;
4295
4296 case INSN_LEA_RIP:
4297 if (!insn_reloc(file, insn)) {
4298 /* local function pointer reference without reloc */
4299
4300 off = arch_jump_destination(insn);
4301
4302 dest = find_insn(file, insn->sec, off);
4303 if (!dest) {
4304 WARN_INSN(insn, "corrupt function pointer reference");
4305 return 1;
4306 }
4307
4308 return __validate_ibt_insn(file, insn, dest);
4309 }
4310 break;
4311
4312 default:
4313 break;
4314 }
4315
4316 for (reloc = insn_reloc(file, insn);
4317 reloc;
4318 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4319 reloc_offset(reloc) + 1,
4320 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4321
4322 off = reloc->sym->offset;
4323 if (reloc_type(reloc) == R_X86_64_PC32 ||
4324 reloc_type(reloc) == R_X86_64_PLT32)
4325 off += arch_dest_reloc_offset(reloc_addend(reloc));
4326 else
4327 off += reloc_addend(reloc);
4328
4329 dest = find_insn(file, reloc->sym->sec, off);
4330 if (!dest)
4331 continue;
4332
4333 warnings += __validate_ibt_insn(file, insn, dest);
4334 }
4335
4336 return warnings;
4337 }
4338
validate_ibt_data_reloc(struct objtool_file * file,struct reloc * reloc)4339 static int validate_ibt_data_reloc(struct objtool_file *file,
4340 struct reloc *reloc)
4341 {
4342 struct instruction *dest;
4343
4344 dest = find_insn(file, reloc->sym->sec,
4345 reloc->sym->offset + reloc_addend(reloc));
4346 if (!dest)
4347 return 0;
4348
4349 if (dest->type == INSN_ENDBR) {
4350 mark_endbr_used(dest);
4351 return 0;
4352 }
4353
4354 if (dest->noendbr)
4355 return 0;
4356
4357 WARN_FUNC("data relocation to !ENDBR: %s",
4358 reloc->sec->base, reloc_offset(reloc),
4359 offstr(dest->sec, dest->offset));
4360
4361 return 1;
4362 }
4363
4364 /*
4365 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4366 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4367 * NOPs) later, in create_ibt_endbr_seal_sections().
4368 */
validate_ibt(struct objtool_file * file)4369 static int validate_ibt(struct objtool_file *file)
4370 {
4371 struct section *sec;
4372 struct reloc *reloc;
4373 struct instruction *insn;
4374 int warnings = 0;
4375
4376 for_each_insn(file, insn)
4377 warnings += validate_ibt_insn(file, insn);
4378
4379 for_each_sec(file, sec) {
4380
4381 /* Already done by validate_ibt_insn() */
4382 if (sec->sh.sh_flags & SHF_EXECINSTR)
4383 continue;
4384
4385 if (!sec->rsec)
4386 continue;
4387
4388 /*
4389 * These sections can reference text addresses, but not with
4390 * the intent to indirect branch to them.
4391 */
4392 if ((!strncmp(sec->name, ".discard", 8) &&
4393 strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
4394 !strncmp(sec->name, ".debug", 6) ||
4395 !strcmp(sec->name, ".altinstructions") ||
4396 !strcmp(sec->name, ".ibt_endbr_seal") ||
4397 !strcmp(sec->name, ".orc_unwind_ip") ||
4398 !strcmp(sec->name, ".parainstructions") ||
4399 !strcmp(sec->name, ".retpoline_sites") ||
4400 !strcmp(sec->name, ".smp_locks") ||
4401 !strcmp(sec->name, ".static_call_sites") ||
4402 !strcmp(sec->name, "_error_injection_whitelist") ||
4403 !strcmp(sec->name, "_kprobe_blacklist") ||
4404 !strcmp(sec->name, "__bug_table") ||
4405 !strcmp(sec->name, "__ex_table") ||
4406 !strcmp(sec->name, "__jump_table") ||
4407 !strcmp(sec->name, "__mcount_loc") ||
4408 !strcmp(sec->name, ".kcfi_traps") ||
4409 !strcmp(sec->name, ".llvm.call-graph-profile") ||
4410 !strcmp(sec->name, ".llvm_bb_addr_map") ||
4411 !strcmp(sec->name, "__tracepoints") ||
4412 strstr(sec->name, "__patchable_function_entries"))
4413 continue;
4414
4415 for_each_reloc(sec->rsec, reloc)
4416 warnings += validate_ibt_data_reloc(file, reloc);
4417 }
4418
4419 return warnings;
4420 }
4421
validate_sls(struct objtool_file * file)4422 static int validate_sls(struct objtool_file *file)
4423 {
4424 struct instruction *insn, *next_insn;
4425 int warnings = 0;
4426
4427 for_each_insn(file, insn) {
4428 next_insn = next_insn_same_sec(file, insn);
4429
4430 if (insn->retpoline_safe)
4431 continue;
4432
4433 switch (insn->type) {
4434 case INSN_RETURN:
4435 if (!next_insn || next_insn->type != INSN_TRAP) {
4436 WARN_INSN(insn, "missing int3 after ret");
4437 warnings++;
4438 }
4439
4440 break;
4441 case INSN_JUMP_DYNAMIC:
4442 if (!next_insn || next_insn->type != INSN_TRAP) {
4443 WARN_INSN(insn, "missing int3 after indirect jump");
4444 warnings++;
4445 }
4446 break;
4447 default:
4448 break;
4449 }
4450 }
4451
4452 return warnings;
4453 }
4454
validate_reachable_instructions(struct objtool_file * file)4455 static int validate_reachable_instructions(struct objtool_file *file)
4456 {
4457 struct instruction *insn, *prev_insn;
4458 struct symbol *call_dest;
4459 int warnings = 0;
4460
4461 if (file->ignore_unreachables)
4462 return 0;
4463
4464 for_each_insn(file, insn) {
4465 if (insn->visited || ignore_unreachable_insn(file, insn))
4466 continue;
4467
4468 prev_insn = prev_insn_same_sec(file, insn);
4469 if (prev_insn && prev_insn->dead_end) {
4470 call_dest = insn_call_dest(prev_insn);
4471 if (call_dest) {
4472 WARN_INSN(insn, "%s() missing __noreturn in .c/.h or NORETURN() in noreturns.h",
4473 call_dest->name);
4474 warnings++;
4475 continue;
4476 }
4477 }
4478
4479 WARN_INSN(insn, "unreachable instruction");
4480 warnings++;
4481 }
4482
4483 return warnings;
4484 }
4485
4486 /* 'funcs' is a space-separated list of function names */
disas_funcs(const char * funcs)4487 static int disas_funcs(const char *funcs)
4488 {
4489 const char *objdump_str, *cross_compile;
4490 int size, ret;
4491 char *cmd;
4492
4493 cross_compile = getenv("CROSS_COMPILE");
4494
4495 objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4496 "BEGIN { split(_funcs, funcs); }"
4497 "/^$/ { func_match = 0; }"
4498 "/<.*>:/ { "
4499 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4500 "for (i in funcs) {"
4501 "if (funcs[i] == f) {"
4502 "func_match = 1;"
4503 "base = strtonum(\"0x\" $1);"
4504 "break;"
4505 "}"
4506 "}"
4507 "}"
4508 "{"
4509 "if (func_match) {"
4510 "addr = strtonum(\"0x\" $1);"
4511 "printf(\"%%04x \", addr - base);"
4512 "print;"
4513 "}"
4514 "}' 1>&2";
4515
4516 /* fake snprintf() to calculate the size */
4517 size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4518 if (size <= 0) {
4519 WARN("objdump string size calculation failed");
4520 return -1;
4521 }
4522
4523 cmd = malloc(size);
4524
4525 /* real snprintf() */
4526 snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4527 ret = system(cmd);
4528 if (ret) {
4529 WARN("disassembly failed: %d", ret);
4530 return -1;
4531 }
4532
4533 return 0;
4534 }
4535
disas_warned_funcs(struct objtool_file * file)4536 static int disas_warned_funcs(struct objtool_file *file)
4537 {
4538 struct symbol *sym;
4539 char *funcs = NULL, *tmp;
4540
4541 for_each_sym(file, sym) {
4542 if (sym->warnings) {
4543 if (!funcs) {
4544 funcs = malloc(strlen(sym->name) + 1);
4545 strcpy(funcs, sym->name);
4546 } else {
4547 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4548 sprintf(tmp, "%s %s", funcs, sym->name);
4549 free(funcs);
4550 funcs = tmp;
4551 }
4552 }
4553 }
4554
4555 if (funcs)
4556 disas_funcs(funcs);
4557
4558 return 0;
4559 }
4560
4561 struct insn_chunk {
4562 void *addr;
4563 struct insn_chunk *next;
4564 };
4565
4566 /*
4567 * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4568 * which can trigger more allocations for .debug_* sections whose data hasn't
4569 * been read yet.
4570 */
free_insns(struct objtool_file * file)4571 static void free_insns(struct objtool_file *file)
4572 {
4573 struct instruction *insn;
4574 struct insn_chunk *chunks = NULL, *chunk;
4575
4576 for_each_insn(file, insn) {
4577 if (!insn->idx) {
4578 chunk = malloc(sizeof(*chunk));
4579 chunk->addr = insn;
4580 chunk->next = chunks;
4581 chunks = chunk;
4582 }
4583 }
4584
4585 for (chunk = chunks; chunk; chunk = chunk->next)
4586 free(chunk->addr);
4587 }
4588
check(struct objtool_file * file)4589 int check(struct objtool_file *file)
4590 {
4591 int ret, warnings = 0;
4592
4593 arch_initial_func_cfi_state(&initial_func_cfi);
4594 init_cfi_state(&init_cfi);
4595 init_cfi_state(&func_cfi);
4596 set_func_state(&func_cfi);
4597 init_cfi_state(&force_undefined_cfi);
4598 force_undefined_cfi.force_undefined = true;
4599
4600 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) {
4601 ret = -1;
4602 goto out;
4603 }
4604
4605 cfi_hash_add(&init_cfi);
4606 cfi_hash_add(&func_cfi);
4607
4608 ret = decode_sections(file);
4609 if (ret < 0)
4610 goto out;
4611
4612 warnings += ret;
4613
4614 if (!nr_insns)
4615 goto out;
4616
4617 if (opts.retpoline) {
4618 ret = validate_retpoline(file);
4619 if (ret < 0)
4620 goto out;
4621 warnings += ret;
4622 }
4623
4624 if (opts.stackval || opts.orc || opts.uaccess) {
4625 ret = validate_functions(file);
4626 if (ret < 0)
4627 goto out;
4628 warnings += ret;
4629
4630 ret = validate_unwind_hints(file, NULL);
4631 if (ret < 0)
4632 goto out;
4633 warnings += ret;
4634
4635 if (!warnings) {
4636 ret = validate_reachable_instructions(file);
4637 if (ret < 0)
4638 goto out;
4639 warnings += ret;
4640 }
4641
4642 } else if (opts.noinstr) {
4643 ret = validate_noinstr_sections(file);
4644 if (ret < 0)
4645 goto out;
4646 warnings += ret;
4647 }
4648
4649 if (opts.unret) {
4650 /*
4651 * Must be after validate_branch() and friends, it plays
4652 * further games with insn->visited.
4653 */
4654 ret = validate_unrets(file);
4655 if (ret < 0)
4656 goto out;
4657 warnings += ret;
4658 }
4659
4660 if (opts.ibt) {
4661 ret = validate_ibt(file);
4662 if (ret < 0)
4663 goto out;
4664 warnings += ret;
4665 }
4666
4667 if (opts.sls) {
4668 ret = validate_sls(file);
4669 if (ret < 0)
4670 goto out;
4671 warnings += ret;
4672 }
4673
4674 if (opts.static_call) {
4675 ret = create_static_call_sections(file);
4676 if (ret < 0)
4677 goto out;
4678 warnings += ret;
4679 }
4680
4681 if (opts.retpoline) {
4682 ret = create_retpoline_sites_sections(file);
4683 if (ret < 0)
4684 goto out;
4685 warnings += ret;
4686 }
4687
4688 if (opts.cfi) {
4689 ret = create_cfi_sections(file);
4690 if (ret < 0)
4691 goto out;
4692 warnings += ret;
4693 }
4694
4695 if (opts.rethunk) {
4696 ret = create_return_sites_sections(file);
4697 if (ret < 0)
4698 goto out;
4699 warnings += ret;
4700
4701 if (opts.hack_skylake) {
4702 ret = create_direct_call_sections(file);
4703 if (ret < 0)
4704 goto out;
4705 warnings += ret;
4706 }
4707 }
4708
4709 if (opts.mcount) {
4710 ret = create_mcount_loc_sections(file);
4711 if (ret < 0)
4712 goto out;
4713 warnings += ret;
4714 }
4715
4716 if (opts.prefix) {
4717 ret = add_prefix_symbols(file);
4718 if (ret < 0)
4719 goto out;
4720 warnings += ret;
4721 }
4722
4723 if (opts.ibt) {
4724 ret = create_ibt_endbr_seal_sections(file);
4725 if (ret < 0)
4726 goto out;
4727 warnings += ret;
4728 }
4729
4730 if (opts.orc && nr_insns) {
4731 ret = orc_create(file);
4732 if (ret < 0)
4733 goto out;
4734 warnings += ret;
4735 }
4736
4737 free_insns(file);
4738
4739 if (opts.verbose)
4740 disas_warned_funcs(file);
4741
4742 if (opts.stats) {
4743 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4744 printf("nr_cfi: %ld\n", nr_cfi);
4745 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4746 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4747 }
4748
4749 out:
4750 /*
4751 * CONFIG_OBJTOOL_WERROR upgrades all warnings (and errors) to actual
4752 * errors.
4753 *
4754 * Note that even "fatal" type errors don't actually return an error
4755 * without CONFIG_OBJTOOL_WERROR. That probably needs improved at some
4756 * point.
4757 */
4758 if (opts.werror && (ret || warnings)) {
4759 if (warnings)
4760 WARN("%d warning(s) upgraded to errors", warnings);
4761 return 1;
4762 }
4763
4764 return 0;
4765 }
4766