1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4 */
5
6 #include <string.h>
7 #include <stdlib.h>
8 #include <inttypes.h>
9 #include <sys/mman.h>
10
11 #include <objtool/builtin.h>
12 #include <objtool/cfi.h>
13 #include <objtool/arch.h>
14 #include <objtool/check.h>
15 #include <objtool/special.h>
16 #include <objtool/warn.h>
17 #include <objtool/endianness.h>
18
19 #include <linux/objtool_types.h>
20 #include <linux/hashtable.h>
21 #include <linux/kernel.h>
22 #include <linux/static_call_types.h>
23 #include <linux/string.h>
24
25 struct alternative {
26 struct alternative *next;
27 struct instruction *insn;
28 };
29
30 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
31
32 static struct cfi_init_state initial_func_cfi;
33 static struct cfi_state init_cfi;
34 static struct cfi_state func_cfi;
35 static struct cfi_state force_undefined_cfi;
36
find_insn(struct objtool_file * file,struct section * sec,unsigned long offset)37 struct instruction *find_insn(struct objtool_file *file,
38 struct section *sec, unsigned long offset)
39 {
40 struct instruction *insn;
41
42 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
43 if (insn->sec == sec && insn->offset == offset)
44 return insn;
45 }
46
47 return NULL;
48 }
49
next_insn_same_sec(struct objtool_file * file,struct instruction * insn)50 struct instruction *next_insn_same_sec(struct objtool_file *file,
51 struct instruction *insn)
52 {
53 if (insn->idx == INSN_CHUNK_MAX)
54 return find_insn(file, insn->sec, insn->offset + insn->len);
55
56 insn++;
57 if (!insn->len)
58 return NULL;
59
60 return insn;
61 }
62
next_insn_same_func(struct objtool_file * file,struct instruction * insn)63 static struct instruction *next_insn_same_func(struct objtool_file *file,
64 struct instruction *insn)
65 {
66 struct instruction *next = next_insn_same_sec(file, insn);
67 struct symbol *func = insn_func(insn);
68
69 if (!func)
70 return NULL;
71
72 if (next && insn_func(next) == func)
73 return next;
74
75 /* Check if we're already in the subfunction: */
76 if (func == func->cfunc)
77 return NULL;
78
79 /* Move to the subfunction: */
80 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
81 }
82
prev_insn_same_sec(struct objtool_file * file,struct instruction * insn)83 static struct instruction *prev_insn_same_sec(struct objtool_file *file,
84 struct instruction *insn)
85 {
86 if (insn->idx == 0) {
87 if (insn->prev_len)
88 return find_insn(file, insn->sec, insn->offset - insn->prev_len);
89 return NULL;
90 }
91
92 return insn - 1;
93 }
94
prev_insn_same_sym(struct objtool_file * file,struct instruction * insn)95 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
96 struct instruction *insn)
97 {
98 struct instruction *prev = prev_insn_same_sec(file, insn);
99
100 if (prev && insn_func(prev) == insn_func(insn))
101 return prev;
102
103 return NULL;
104 }
105
106 #define for_each_insn(file, insn) \
107 for (struct section *__sec, *__fake = (struct section *)1; \
108 __fake; __fake = NULL) \
109 for_each_sec(file, __sec) \
110 sec_for_each_insn(file, __sec, insn)
111
112 #define func_for_each_insn(file, func, insn) \
113 for (insn = find_insn(file, func->sec, func->offset); \
114 insn; \
115 insn = next_insn_same_func(file, insn))
116
117 #define sym_for_each_insn(file, sym, insn) \
118 for (insn = find_insn(file, sym->sec, sym->offset); \
119 insn && insn->offset < sym->offset + sym->len; \
120 insn = next_insn_same_sec(file, insn))
121
122 #define sym_for_each_insn_continue_reverse(file, sym, insn) \
123 for (insn = prev_insn_same_sec(file, insn); \
124 insn && insn->offset >= sym->offset; \
125 insn = prev_insn_same_sec(file, insn))
126
127 #define sec_for_each_insn_from(file, insn) \
128 for (; insn; insn = next_insn_same_sec(file, insn))
129
130 #define sec_for_each_insn_continue(file, insn) \
131 for (insn = next_insn_same_sec(file, insn); insn; \
132 insn = next_insn_same_sec(file, insn))
133
insn_call_dest(struct instruction * insn)134 static inline struct symbol *insn_call_dest(struct instruction *insn)
135 {
136 if (insn->type == INSN_JUMP_DYNAMIC ||
137 insn->type == INSN_CALL_DYNAMIC)
138 return NULL;
139
140 return insn->_call_dest;
141 }
142
insn_jump_table(struct instruction * insn)143 static inline struct reloc *insn_jump_table(struct instruction *insn)
144 {
145 if (insn->type == INSN_JUMP_DYNAMIC ||
146 insn->type == INSN_CALL_DYNAMIC)
147 return insn->_jump_table;
148
149 return NULL;
150 }
151
insn_jump_table_size(struct instruction * insn)152 static inline unsigned long insn_jump_table_size(struct instruction *insn)
153 {
154 if (insn->type == INSN_JUMP_DYNAMIC ||
155 insn->type == INSN_CALL_DYNAMIC)
156 return insn->_jump_table_size;
157
158 return 0;
159 }
160
is_jump_table_jump(struct instruction * insn)161 static bool is_jump_table_jump(struct instruction *insn)
162 {
163 struct alt_group *alt_group = insn->alt_group;
164
165 if (insn_jump_table(insn))
166 return true;
167
168 /* Retpoline alternative for a jump table? */
169 return alt_group && alt_group->orig_group &&
170 insn_jump_table(alt_group->orig_group->first_insn);
171 }
172
is_sibling_call(struct instruction * insn)173 static bool is_sibling_call(struct instruction *insn)
174 {
175 /*
176 * Assume only STT_FUNC calls have jump-tables.
177 */
178 if (insn_func(insn)) {
179 /* An indirect jump is either a sibling call or a jump to a table. */
180 if (insn->type == INSN_JUMP_DYNAMIC)
181 return !is_jump_table_jump(insn);
182 }
183
184 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
185 return (is_static_jump(insn) && insn_call_dest(insn));
186 }
187
188 /*
189 * Checks if a string ends with another.
190 */
str_ends_with(const char * s,const char * sub)191 static bool str_ends_with(const char *s, const char *sub)
192 {
193 const int slen = strlen(s);
194 const int sublen = strlen(sub);
195
196 if (sublen > slen)
197 return 0;
198
199 return !memcmp(s + slen - sublen, sub, sublen);
200 }
201
202 /*
203 * Checks if a function is a Rust "noreturn" one.
204 */
is_rust_noreturn(const struct symbol * func)205 static bool is_rust_noreturn(const struct symbol *func)
206 {
207 /*
208 * If it does not start with "_R", then it is not a Rust symbol.
209 */
210 if (strncmp(func->name, "_R", 2))
211 return false;
212
213 /*
214 * These are just heuristics -- we do not control the precise symbol
215 * name, due to the crate disambiguators (which depend on the compiler)
216 * as well as changes to the source code itself between versions (since
217 * these come from the Rust standard library).
218 */
219 return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") ||
220 str_ends_with(func->name, "_4core6option13unwrap_failed") ||
221 str_ends_with(func->name, "_4core6result13unwrap_failed") ||
222 str_ends_with(func->name, "_4core9panicking5panic") ||
223 str_ends_with(func->name, "_4core9panicking9panic_fmt") ||
224 str_ends_with(func->name, "_4core9panicking14panic_explicit") ||
225 str_ends_with(func->name, "_4core9panicking14panic_nounwind") ||
226 str_ends_with(func->name, "_4core9panicking18panic_bounds_check") ||
227 str_ends_with(func->name, "_4core9panicking19assert_failed_inner") ||
228 str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference") ||
229 str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") ||
230 str_ends_with(func->name, "_7___rustc17rust_begin_unwind") ||
231 strstr(func->name, "_4core9panicking13assert_failed") ||
232 strstr(func->name, "_4core9panicking11panic_const24panic_const_") ||
233 (strstr(func->name, "_4core5slice5index") &&
234 strstr(func->name, "slice_") &&
235 str_ends_with(func->name, "_fail"));
236 }
237
238 /*
239 * This checks to see if the given function is a "noreturn" function.
240 *
241 * For global functions which are outside the scope of this object file, we
242 * have to keep a manual list of them.
243 *
244 * For local functions, we have to detect them manually by simply looking for
245 * the lack of a return instruction.
246 */
__dead_end_function(struct objtool_file * file,struct symbol * func,int recursion)247 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
248 int recursion)
249 {
250 int i;
251 struct instruction *insn;
252 bool empty = true;
253
254 #define NORETURN(func) __stringify(func),
255 static const char * const global_noreturns[] = {
256 #include "noreturns.h"
257 };
258 #undef NORETURN
259
260 if (!func)
261 return false;
262
263 if (func->bind == STB_GLOBAL || func->bind == STB_WEAK) {
264 if (is_rust_noreturn(func))
265 return true;
266
267 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
268 if (!strcmp(func->name, global_noreturns[i]))
269 return true;
270 }
271
272 if (func->bind == STB_WEAK)
273 return false;
274
275 if (!func->len)
276 return false;
277
278 insn = find_insn(file, func->sec, func->offset);
279 if (!insn || !insn_func(insn))
280 return false;
281
282 func_for_each_insn(file, func, insn) {
283 empty = false;
284
285 if (insn->type == INSN_RETURN)
286 return false;
287 }
288
289 if (empty)
290 return false;
291
292 /*
293 * A function can have a sibling call instead of a return. In that
294 * case, the function's dead-end status depends on whether the target
295 * of the sibling call returns.
296 */
297 func_for_each_insn(file, func, insn) {
298 if (is_sibling_call(insn)) {
299 struct instruction *dest = insn->jump_dest;
300
301 if (!dest)
302 /* sibling call to another file */
303 return false;
304
305 /* local sibling call */
306 if (recursion == 5) {
307 /*
308 * Infinite recursion: two functions have
309 * sibling calls to each other. This is a very
310 * rare case. It means they aren't dead ends.
311 */
312 return false;
313 }
314
315 return __dead_end_function(file, insn_func(dest), recursion+1);
316 }
317 }
318
319 return true;
320 }
321
dead_end_function(struct objtool_file * file,struct symbol * func)322 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
323 {
324 return __dead_end_function(file, func, 0);
325 }
326
init_cfi_state(struct cfi_state * cfi)327 static void init_cfi_state(struct cfi_state *cfi)
328 {
329 int i;
330
331 for (i = 0; i < CFI_NUM_REGS; i++) {
332 cfi->regs[i].base = CFI_UNDEFINED;
333 cfi->vals[i].base = CFI_UNDEFINED;
334 }
335 cfi->cfa.base = CFI_UNDEFINED;
336 cfi->drap_reg = CFI_UNDEFINED;
337 cfi->drap_offset = -1;
338 }
339
init_insn_state(struct objtool_file * file,struct insn_state * state,struct section * sec)340 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
341 struct section *sec)
342 {
343 memset(state, 0, sizeof(*state));
344 init_cfi_state(&state->cfi);
345
346 if (opts.noinstr && sec)
347 state->noinstr = sec->noinstr;
348 }
349
cfi_alloc(void)350 static struct cfi_state *cfi_alloc(void)
351 {
352 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
353 if (!cfi) {
354 ERROR_GLIBC("calloc");
355 exit(1);
356 }
357 nr_cfi++;
358 return cfi;
359 }
360
361 static int cfi_bits;
362 static struct hlist_head *cfi_hash;
363
cficmp(struct cfi_state * cfi1,struct cfi_state * cfi2)364 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
365 {
366 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
367 (void *)cfi2 + sizeof(cfi2->hash),
368 sizeof(struct cfi_state) - sizeof(struct hlist_node));
369 }
370
cfi_key(struct cfi_state * cfi)371 static inline u32 cfi_key(struct cfi_state *cfi)
372 {
373 return jhash((void *)cfi + sizeof(cfi->hash),
374 sizeof(*cfi) - sizeof(cfi->hash), 0);
375 }
376
cfi_hash_find_or_add(struct cfi_state * cfi)377 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
378 {
379 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
380 struct cfi_state *obj;
381
382 hlist_for_each_entry(obj, head, hash) {
383 if (!cficmp(cfi, obj)) {
384 nr_cfi_cache++;
385 return obj;
386 }
387 }
388
389 obj = cfi_alloc();
390 *obj = *cfi;
391 hlist_add_head(&obj->hash, head);
392
393 return obj;
394 }
395
cfi_hash_add(struct cfi_state * cfi)396 static void cfi_hash_add(struct cfi_state *cfi)
397 {
398 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
399
400 hlist_add_head(&cfi->hash, head);
401 }
402
cfi_hash_alloc(unsigned long size)403 static void *cfi_hash_alloc(unsigned long size)
404 {
405 cfi_bits = max(10, ilog2(size));
406 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
407 PROT_READ|PROT_WRITE,
408 MAP_PRIVATE|MAP_ANON, -1, 0);
409 if (cfi_hash == (void *)-1L) {
410 ERROR_GLIBC("mmap fail cfi_hash");
411 cfi_hash = NULL;
412 } else if (opts.stats) {
413 printf("cfi_bits: %d\n", cfi_bits);
414 }
415
416 return cfi_hash;
417 }
418
419 static unsigned long nr_insns;
420 static unsigned long nr_insns_visited;
421
422 /*
423 * Call the arch-specific instruction decoder for all the instructions and add
424 * them to the global instruction list.
425 */
decode_instructions(struct objtool_file * file)426 static int decode_instructions(struct objtool_file *file)
427 {
428 struct section *sec;
429 struct symbol *func;
430 unsigned long offset;
431 struct instruction *insn;
432 int ret;
433
434 for_each_sec(file, sec) {
435 struct instruction *insns = NULL;
436 u8 prev_len = 0;
437 u8 idx = 0;
438
439 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
440 continue;
441
442 if (strcmp(sec->name, ".altinstr_replacement") &&
443 strcmp(sec->name, ".altinstr_aux") &&
444 strncmp(sec->name, ".discard.", 9))
445 sec->text = true;
446
447 if (!strcmp(sec->name, ".noinstr.text") ||
448 !strcmp(sec->name, ".entry.text") ||
449 !strcmp(sec->name, ".cpuidle.text") ||
450 !strncmp(sec->name, ".text..__x86.", 13))
451 sec->noinstr = true;
452
453 /*
454 * .init.text code is ran before userspace and thus doesn't
455 * strictly need retpolines, except for modules which are
456 * loaded late, they very much do need retpoline in their
457 * .init.text
458 */
459 if (!strcmp(sec->name, ".init.text") && !opts.module)
460 sec->init = true;
461
462 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
463 if (!insns || idx == INSN_CHUNK_MAX) {
464 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
465 if (!insns) {
466 ERROR_GLIBC("calloc");
467 return -1;
468 }
469 idx = 0;
470 } else {
471 idx++;
472 }
473 insn = &insns[idx];
474 insn->idx = idx;
475
476 INIT_LIST_HEAD(&insn->call_node);
477 insn->sec = sec;
478 insn->offset = offset;
479 insn->prev_len = prev_len;
480
481 ret = arch_decode_instruction(file, sec, offset,
482 sec->sh.sh_size - offset,
483 insn);
484 if (ret)
485 return ret;
486
487 prev_len = insn->len;
488
489 /*
490 * By default, "ud2" is a dead end unless otherwise
491 * annotated, because GCC 7 inserts it for certain
492 * divide-by-zero cases.
493 */
494 if (insn->type == INSN_BUG)
495 insn->dead_end = true;
496
497 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
498 nr_insns++;
499 }
500
501 sec_for_each_sym(sec, func) {
502 if (func->type != STT_NOTYPE && func->type != STT_FUNC)
503 continue;
504
505 if (func->offset == sec->sh.sh_size) {
506 /* Heuristic: likely an "end" symbol */
507 if (func->type == STT_NOTYPE)
508 continue;
509 ERROR("%s(): STT_FUNC at end of section", func->name);
510 return -1;
511 }
512
513 if (func->embedded_insn || func->alias != func)
514 continue;
515
516 if (!find_insn(file, sec, func->offset)) {
517 ERROR("%s(): can't find starting instruction", func->name);
518 return -1;
519 }
520
521 sym_for_each_insn(file, func, insn) {
522 insn->sym = func;
523 if (func->type == STT_FUNC &&
524 insn->type == INSN_ENDBR &&
525 list_empty(&insn->call_node)) {
526 if (insn->offset == func->offset) {
527 list_add_tail(&insn->call_node, &file->endbr_list);
528 file->nr_endbr++;
529 } else {
530 file->nr_endbr_int++;
531 }
532 }
533 }
534 }
535 }
536
537 if (opts.stats)
538 printf("nr_insns: %lu\n", nr_insns);
539
540 return 0;
541 }
542
543 /*
544 * Read the pv_ops[] .data table to find the static initialized values.
545 */
add_pv_ops(struct objtool_file * file,const char * symname)546 static int add_pv_ops(struct objtool_file *file, const char *symname)
547 {
548 struct symbol *sym, *func;
549 unsigned long off, end;
550 struct reloc *reloc;
551 int idx;
552
553 sym = find_symbol_by_name(file->elf, symname);
554 if (!sym)
555 return 0;
556
557 off = sym->offset;
558 end = off + sym->len;
559 for (;;) {
560 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
561 if (!reloc)
562 break;
563
564 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
565
566 func = reloc->sym;
567 if (func->type == STT_SECTION)
568 func = find_symbol_by_offset(reloc->sym->sec,
569 reloc_addend(reloc));
570 if (!func) {
571 ERROR_FUNC(reloc->sym->sec, reloc_addend(reloc),
572 "can't find func at %s[%d]", symname, idx);
573 return -1;
574 }
575
576 if (objtool_pv_add(file, idx, func))
577 return -1;
578
579 off = reloc_offset(reloc) + 1;
580 if (off > end)
581 break;
582 }
583
584 return 0;
585 }
586
587 /*
588 * Allocate and initialize file->pv_ops[].
589 */
init_pv_ops(struct objtool_file * file)590 static int init_pv_ops(struct objtool_file *file)
591 {
592 static const char *pv_ops_tables[] = {
593 "pv_ops",
594 "xen_cpu_ops",
595 "xen_irq_ops",
596 "xen_mmu_ops",
597 NULL,
598 };
599 const char *pv_ops;
600 struct symbol *sym;
601 int idx, nr, ret;
602
603 if (!opts.noinstr)
604 return 0;
605
606 file->pv_ops = NULL;
607
608 sym = find_symbol_by_name(file->elf, "pv_ops");
609 if (!sym)
610 return 0;
611
612 nr = sym->len / sizeof(unsigned long);
613 file->pv_ops = calloc(sizeof(struct pv_state), nr);
614 if (!file->pv_ops) {
615 ERROR_GLIBC("calloc");
616 return -1;
617 }
618
619 for (idx = 0; idx < nr; idx++)
620 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
621
622 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) {
623 ret = add_pv_ops(file, pv_ops);
624 if (ret)
625 return ret;
626 }
627
628 return 0;
629 }
630
create_static_call_sections(struct objtool_file * file)631 static int create_static_call_sections(struct objtool_file *file)
632 {
633 struct static_call_site *site;
634 struct section *sec;
635 struct instruction *insn;
636 struct symbol *key_sym;
637 char *key_name, *tmp;
638 int idx;
639
640 sec = find_section_by_name(file->elf, ".static_call_sites");
641 if (sec) {
642 INIT_LIST_HEAD(&file->static_call_list);
643 WARN("file already has .static_call_sites section, skipping");
644 return 0;
645 }
646
647 if (list_empty(&file->static_call_list))
648 return 0;
649
650 idx = 0;
651 list_for_each_entry(insn, &file->static_call_list, call_node)
652 idx++;
653
654 sec = elf_create_section_pair(file->elf, ".static_call_sites",
655 sizeof(*site), idx, idx * 2);
656 if (!sec)
657 return -1;
658
659 /* Allow modules to modify the low bits of static_call_site::key */
660 sec->sh.sh_flags |= SHF_WRITE;
661
662 idx = 0;
663 list_for_each_entry(insn, &file->static_call_list, call_node) {
664
665 /* populate reloc for 'addr' */
666 if (!elf_init_reloc_text_sym(file->elf, sec,
667 idx * sizeof(*site), idx * 2,
668 insn->sec, insn->offset))
669 return -1;
670
671 /* find key symbol */
672 key_name = strdup(insn_call_dest(insn)->name);
673 if (!key_name) {
674 ERROR_GLIBC("strdup");
675 return -1;
676 }
677 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
678 STATIC_CALL_TRAMP_PREFIX_LEN)) {
679 ERROR("static_call: trampoline name malformed: %s", key_name);
680 return -1;
681 }
682 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
683 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
684
685 key_sym = find_symbol_by_name(file->elf, tmp);
686 if (!key_sym) {
687 if (!opts.module) {
688 ERROR("static_call: can't find static_call_key symbol: %s", tmp);
689 return -1;
690 }
691
692 /*
693 * For modules(), the key might not be exported, which
694 * means the module can make static calls but isn't
695 * allowed to change them.
696 *
697 * In that case we temporarily set the key to be the
698 * trampoline address. This is fixed up in
699 * static_call_add_module().
700 */
701 key_sym = insn_call_dest(insn);
702 }
703
704 /* populate reloc for 'key' */
705 if (!elf_init_reloc_data_sym(file->elf, sec,
706 idx * sizeof(*site) + 4,
707 (idx * 2) + 1, key_sym,
708 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
709 return -1;
710
711 idx++;
712 }
713
714 return 0;
715 }
716
create_retpoline_sites_sections(struct objtool_file * file)717 static int create_retpoline_sites_sections(struct objtool_file *file)
718 {
719 struct instruction *insn;
720 struct section *sec;
721 int idx;
722
723 sec = find_section_by_name(file->elf, ".retpoline_sites");
724 if (sec) {
725 WARN("file already has .retpoline_sites, skipping");
726 return 0;
727 }
728
729 idx = 0;
730 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
731 idx++;
732
733 if (!idx)
734 return 0;
735
736 sec = elf_create_section_pair(file->elf, ".retpoline_sites",
737 sizeof(int), idx, idx);
738 if (!sec)
739 return -1;
740
741 idx = 0;
742 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
743
744 if (!elf_init_reloc_text_sym(file->elf, sec,
745 idx * sizeof(int), idx,
746 insn->sec, insn->offset))
747 return -1;
748
749 idx++;
750 }
751
752 return 0;
753 }
754
create_return_sites_sections(struct objtool_file * file)755 static int create_return_sites_sections(struct objtool_file *file)
756 {
757 struct instruction *insn;
758 struct section *sec;
759 int idx;
760
761 sec = find_section_by_name(file->elf, ".return_sites");
762 if (sec) {
763 WARN("file already has .return_sites, skipping");
764 return 0;
765 }
766
767 idx = 0;
768 list_for_each_entry(insn, &file->return_thunk_list, call_node)
769 idx++;
770
771 if (!idx)
772 return 0;
773
774 sec = elf_create_section_pair(file->elf, ".return_sites",
775 sizeof(int), idx, idx);
776 if (!sec)
777 return -1;
778
779 idx = 0;
780 list_for_each_entry(insn, &file->return_thunk_list, call_node) {
781
782 if (!elf_init_reloc_text_sym(file->elf, sec,
783 idx * sizeof(int), idx,
784 insn->sec, insn->offset))
785 return -1;
786
787 idx++;
788 }
789
790 return 0;
791 }
792
create_ibt_endbr_seal_sections(struct objtool_file * file)793 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
794 {
795 struct instruction *insn;
796 struct section *sec;
797 int idx;
798
799 sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
800 if (sec) {
801 WARN("file already has .ibt_endbr_seal, skipping");
802 return 0;
803 }
804
805 idx = 0;
806 list_for_each_entry(insn, &file->endbr_list, call_node)
807 idx++;
808
809 if (opts.stats) {
810 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
811 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
812 printf("ibt: superfluous ENDBR: %d\n", idx);
813 }
814
815 if (!idx)
816 return 0;
817
818 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
819 sizeof(int), idx, idx);
820 if (!sec)
821 return -1;
822
823 idx = 0;
824 list_for_each_entry(insn, &file->endbr_list, call_node) {
825
826 int *site = (int *)sec->data->d_buf + idx;
827 struct symbol *sym = insn->sym;
828 *site = 0;
829
830 if (opts.module && sym && sym->type == STT_FUNC &&
831 insn->offset == sym->offset &&
832 (!strcmp(sym->name, "init_module") ||
833 !strcmp(sym->name, "cleanup_module"))) {
834 ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead",
835 sym->name);
836 return -1;
837 }
838
839 if (!elf_init_reloc_text_sym(file->elf, sec,
840 idx * sizeof(int), idx,
841 insn->sec, insn->offset))
842 return -1;
843
844 idx++;
845 }
846
847 return 0;
848 }
849
create_cfi_sections(struct objtool_file * file)850 static int create_cfi_sections(struct objtool_file *file)
851 {
852 struct section *sec;
853 struct symbol *sym;
854 int idx;
855
856 sec = find_section_by_name(file->elf, ".cfi_sites");
857 if (sec) {
858 INIT_LIST_HEAD(&file->call_list);
859 WARN("file already has .cfi_sites section, skipping");
860 return 0;
861 }
862
863 idx = 0;
864 for_each_sym(file, sym) {
865 if (sym->type != STT_FUNC)
866 continue;
867
868 if (strncmp(sym->name, "__cfi_", 6))
869 continue;
870
871 idx++;
872 }
873
874 sec = elf_create_section_pair(file->elf, ".cfi_sites",
875 sizeof(unsigned int), idx, idx);
876 if (!sec)
877 return -1;
878
879 idx = 0;
880 for_each_sym(file, sym) {
881 if (sym->type != STT_FUNC)
882 continue;
883
884 if (strncmp(sym->name, "__cfi_", 6))
885 continue;
886
887 if (!elf_init_reloc_text_sym(file->elf, sec,
888 idx * sizeof(unsigned int), idx,
889 sym->sec, sym->offset))
890 return -1;
891
892 idx++;
893 }
894
895 return 0;
896 }
897
create_mcount_loc_sections(struct objtool_file * file)898 static int create_mcount_loc_sections(struct objtool_file *file)
899 {
900 size_t addr_size = elf_addr_size(file->elf);
901 struct instruction *insn;
902 struct section *sec;
903 int idx;
904
905 sec = find_section_by_name(file->elf, "__mcount_loc");
906 if (sec) {
907 INIT_LIST_HEAD(&file->mcount_loc_list);
908 WARN("file already has __mcount_loc section, skipping");
909 return 0;
910 }
911
912 if (list_empty(&file->mcount_loc_list))
913 return 0;
914
915 idx = 0;
916 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
917 idx++;
918
919 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
920 idx, idx);
921 if (!sec)
922 return -1;
923
924 sec->sh.sh_addralign = addr_size;
925
926 idx = 0;
927 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
928
929 struct reloc *reloc;
930
931 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
932 insn->sec, insn->offset);
933 if (!reloc)
934 return -1;
935
936 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
937
938 idx++;
939 }
940
941 return 0;
942 }
943
create_direct_call_sections(struct objtool_file * file)944 static int create_direct_call_sections(struct objtool_file *file)
945 {
946 struct instruction *insn;
947 struct section *sec;
948 int idx;
949
950 sec = find_section_by_name(file->elf, ".call_sites");
951 if (sec) {
952 INIT_LIST_HEAD(&file->call_list);
953 WARN("file already has .call_sites section, skipping");
954 return 0;
955 }
956
957 if (list_empty(&file->call_list))
958 return 0;
959
960 idx = 0;
961 list_for_each_entry(insn, &file->call_list, call_node)
962 idx++;
963
964 sec = elf_create_section_pair(file->elf, ".call_sites",
965 sizeof(unsigned int), idx, idx);
966 if (!sec)
967 return -1;
968
969 idx = 0;
970 list_for_each_entry(insn, &file->call_list, call_node) {
971
972 if (!elf_init_reloc_text_sym(file->elf, sec,
973 idx * sizeof(unsigned int), idx,
974 insn->sec, insn->offset))
975 return -1;
976
977 idx++;
978 }
979
980 return 0;
981 }
982
983 /*
984 * Warnings shouldn't be reported for ignored functions.
985 */
add_ignores(struct objtool_file * file)986 static int add_ignores(struct objtool_file *file)
987 {
988 struct section *rsec;
989 struct symbol *func;
990 struct reloc *reloc;
991
992 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
993 if (!rsec)
994 return 0;
995
996 for_each_reloc(rsec, reloc) {
997 switch (reloc->sym->type) {
998 case STT_FUNC:
999 func = reloc->sym;
1000 break;
1001
1002 case STT_SECTION:
1003 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1004 if (!func)
1005 continue;
1006 break;
1007
1008 default:
1009 ERROR("unexpected relocation symbol type in %s: %d",
1010 rsec->name, reloc->sym->type);
1011 return -1;
1012 }
1013
1014 func->ignore = true;
1015 if (func->cfunc)
1016 func->cfunc->ignore = true;
1017 }
1018
1019 return 0;
1020 }
1021
1022 /*
1023 * This is a whitelist of functions that is allowed to be called with AC set.
1024 * The list is meant to be minimal and only contains compiler instrumentation
1025 * ABI and a few functions used to implement *_{to,from}_user() functions.
1026 *
1027 * These functions must not directly change AC, but may PUSHF/POPF.
1028 */
1029 static const char *uaccess_safe_builtin[] = {
1030 /* KASAN */
1031 "kasan_report",
1032 "kasan_check_range",
1033 /* KASAN out-of-line */
1034 "__asan_loadN_noabort",
1035 "__asan_load1_noabort",
1036 "__asan_load2_noabort",
1037 "__asan_load4_noabort",
1038 "__asan_load8_noabort",
1039 "__asan_load16_noabort",
1040 "__asan_storeN_noabort",
1041 "__asan_store1_noabort",
1042 "__asan_store2_noabort",
1043 "__asan_store4_noabort",
1044 "__asan_store8_noabort",
1045 "__asan_store16_noabort",
1046 "__kasan_check_read",
1047 "__kasan_check_write",
1048 /* KASAN in-line */
1049 "__asan_report_load_n_noabort",
1050 "__asan_report_load1_noabort",
1051 "__asan_report_load2_noabort",
1052 "__asan_report_load4_noabort",
1053 "__asan_report_load8_noabort",
1054 "__asan_report_load16_noabort",
1055 "__asan_report_store_n_noabort",
1056 "__asan_report_store1_noabort",
1057 "__asan_report_store2_noabort",
1058 "__asan_report_store4_noabort",
1059 "__asan_report_store8_noabort",
1060 "__asan_report_store16_noabort",
1061 /* KCSAN */
1062 "__kcsan_check_access",
1063 "__kcsan_mb",
1064 "__kcsan_wmb",
1065 "__kcsan_rmb",
1066 "__kcsan_release",
1067 "kcsan_found_watchpoint",
1068 "kcsan_setup_watchpoint",
1069 "kcsan_check_scoped_accesses",
1070 "kcsan_disable_current",
1071 "kcsan_enable_current_nowarn",
1072 /* KCSAN/TSAN */
1073 "__tsan_func_entry",
1074 "__tsan_func_exit",
1075 "__tsan_read_range",
1076 "__tsan_write_range",
1077 "__tsan_read1",
1078 "__tsan_read2",
1079 "__tsan_read4",
1080 "__tsan_read8",
1081 "__tsan_read16",
1082 "__tsan_write1",
1083 "__tsan_write2",
1084 "__tsan_write4",
1085 "__tsan_write8",
1086 "__tsan_write16",
1087 "__tsan_read_write1",
1088 "__tsan_read_write2",
1089 "__tsan_read_write4",
1090 "__tsan_read_write8",
1091 "__tsan_read_write16",
1092 "__tsan_volatile_read1",
1093 "__tsan_volatile_read2",
1094 "__tsan_volatile_read4",
1095 "__tsan_volatile_read8",
1096 "__tsan_volatile_read16",
1097 "__tsan_volatile_write1",
1098 "__tsan_volatile_write2",
1099 "__tsan_volatile_write4",
1100 "__tsan_volatile_write8",
1101 "__tsan_volatile_write16",
1102 "__tsan_atomic8_load",
1103 "__tsan_atomic16_load",
1104 "__tsan_atomic32_load",
1105 "__tsan_atomic64_load",
1106 "__tsan_atomic8_store",
1107 "__tsan_atomic16_store",
1108 "__tsan_atomic32_store",
1109 "__tsan_atomic64_store",
1110 "__tsan_atomic8_exchange",
1111 "__tsan_atomic16_exchange",
1112 "__tsan_atomic32_exchange",
1113 "__tsan_atomic64_exchange",
1114 "__tsan_atomic8_fetch_add",
1115 "__tsan_atomic16_fetch_add",
1116 "__tsan_atomic32_fetch_add",
1117 "__tsan_atomic64_fetch_add",
1118 "__tsan_atomic8_fetch_sub",
1119 "__tsan_atomic16_fetch_sub",
1120 "__tsan_atomic32_fetch_sub",
1121 "__tsan_atomic64_fetch_sub",
1122 "__tsan_atomic8_fetch_and",
1123 "__tsan_atomic16_fetch_and",
1124 "__tsan_atomic32_fetch_and",
1125 "__tsan_atomic64_fetch_and",
1126 "__tsan_atomic8_fetch_or",
1127 "__tsan_atomic16_fetch_or",
1128 "__tsan_atomic32_fetch_or",
1129 "__tsan_atomic64_fetch_or",
1130 "__tsan_atomic8_fetch_xor",
1131 "__tsan_atomic16_fetch_xor",
1132 "__tsan_atomic32_fetch_xor",
1133 "__tsan_atomic64_fetch_xor",
1134 "__tsan_atomic8_fetch_nand",
1135 "__tsan_atomic16_fetch_nand",
1136 "__tsan_atomic32_fetch_nand",
1137 "__tsan_atomic64_fetch_nand",
1138 "__tsan_atomic8_compare_exchange_strong",
1139 "__tsan_atomic16_compare_exchange_strong",
1140 "__tsan_atomic32_compare_exchange_strong",
1141 "__tsan_atomic64_compare_exchange_strong",
1142 "__tsan_atomic8_compare_exchange_weak",
1143 "__tsan_atomic16_compare_exchange_weak",
1144 "__tsan_atomic32_compare_exchange_weak",
1145 "__tsan_atomic64_compare_exchange_weak",
1146 "__tsan_atomic8_compare_exchange_val",
1147 "__tsan_atomic16_compare_exchange_val",
1148 "__tsan_atomic32_compare_exchange_val",
1149 "__tsan_atomic64_compare_exchange_val",
1150 "__tsan_atomic_thread_fence",
1151 "__tsan_atomic_signal_fence",
1152 "__tsan_unaligned_read16",
1153 "__tsan_unaligned_write16",
1154 /* KCOV */
1155 "write_comp_data",
1156 "check_kcov_mode",
1157 "__sanitizer_cov_trace_pc",
1158 "__sanitizer_cov_trace_const_cmp1",
1159 "__sanitizer_cov_trace_const_cmp2",
1160 "__sanitizer_cov_trace_const_cmp4",
1161 "__sanitizer_cov_trace_const_cmp8",
1162 "__sanitizer_cov_trace_cmp1",
1163 "__sanitizer_cov_trace_cmp2",
1164 "__sanitizer_cov_trace_cmp4",
1165 "__sanitizer_cov_trace_cmp8",
1166 "__sanitizer_cov_trace_switch",
1167 /* KMSAN */
1168 "kmsan_copy_to_user",
1169 "kmsan_disable_current",
1170 "kmsan_enable_current",
1171 "kmsan_report",
1172 "kmsan_unpoison_entry_regs",
1173 "kmsan_unpoison_memory",
1174 "__msan_chain_origin",
1175 "__msan_get_context_state",
1176 "__msan_instrument_asm_store",
1177 "__msan_metadata_ptr_for_load_1",
1178 "__msan_metadata_ptr_for_load_2",
1179 "__msan_metadata_ptr_for_load_4",
1180 "__msan_metadata_ptr_for_load_8",
1181 "__msan_metadata_ptr_for_load_n",
1182 "__msan_metadata_ptr_for_store_1",
1183 "__msan_metadata_ptr_for_store_2",
1184 "__msan_metadata_ptr_for_store_4",
1185 "__msan_metadata_ptr_for_store_8",
1186 "__msan_metadata_ptr_for_store_n",
1187 "__msan_poison_alloca",
1188 "__msan_warning",
1189 /* UBSAN */
1190 "ubsan_type_mismatch_common",
1191 "__ubsan_handle_type_mismatch",
1192 "__ubsan_handle_type_mismatch_v1",
1193 "__ubsan_handle_shift_out_of_bounds",
1194 "__ubsan_handle_load_invalid_value",
1195 /* STACKLEAK */
1196 "stackleak_track_stack",
1197 /* TRACE_BRANCH_PROFILING */
1198 "ftrace_likely_update",
1199 /* STACKPROTECTOR */
1200 "__stack_chk_fail",
1201 /* misc */
1202 "csum_partial_copy_generic",
1203 "copy_mc_fragile",
1204 "copy_mc_fragile_handle_tail",
1205 "copy_mc_enhanced_fast_string",
1206 "rep_stos_alternative",
1207 "rep_movs_alternative",
1208 "__copy_user_nocache",
1209 NULL
1210 };
1211
add_uaccess_safe(struct objtool_file * file)1212 static void add_uaccess_safe(struct objtool_file *file)
1213 {
1214 struct symbol *func;
1215 const char **name;
1216
1217 if (!opts.uaccess)
1218 return;
1219
1220 for (name = uaccess_safe_builtin; *name; name++) {
1221 func = find_symbol_by_name(file->elf, *name);
1222 if (!func)
1223 continue;
1224
1225 func->uaccess_safe = true;
1226 }
1227 }
1228
1229 /*
1230 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1231 * will be added to the .retpoline_sites section.
1232 */
arch_is_retpoline(struct symbol * sym)1233 __weak bool arch_is_retpoline(struct symbol *sym)
1234 {
1235 return false;
1236 }
1237
1238 /*
1239 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1240 * will be added to the .return_sites section.
1241 */
arch_is_rethunk(struct symbol * sym)1242 __weak bool arch_is_rethunk(struct symbol *sym)
1243 {
1244 return false;
1245 }
1246
1247 /*
1248 * Symbols that are embedded inside other instructions, because sometimes crazy
1249 * code exists. These are mostly ignored for validation purposes.
1250 */
arch_is_embedded_insn(struct symbol * sym)1251 __weak bool arch_is_embedded_insn(struct symbol *sym)
1252 {
1253 return false;
1254 }
1255
insn_reloc(struct objtool_file * file,struct instruction * insn)1256 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1257 {
1258 struct reloc *reloc;
1259
1260 if (insn->no_reloc)
1261 return NULL;
1262
1263 if (!file)
1264 return NULL;
1265
1266 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1267 insn->offset, insn->len);
1268 if (!reloc) {
1269 insn->no_reloc = 1;
1270 return NULL;
1271 }
1272
1273 return reloc;
1274 }
1275
remove_insn_ops(struct instruction * insn)1276 static void remove_insn_ops(struct instruction *insn)
1277 {
1278 struct stack_op *op, *next;
1279
1280 for (op = insn->stack_ops; op; op = next) {
1281 next = op->next;
1282 free(op);
1283 }
1284 insn->stack_ops = NULL;
1285 }
1286
annotate_call_site(struct objtool_file * file,struct instruction * insn,bool sibling)1287 static int annotate_call_site(struct objtool_file *file,
1288 struct instruction *insn, bool sibling)
1289 {
1290 struct reloc *reloc = insn_reloc(file, insn);
1291 struct symbol *sym = insn_call_dest(insn);
1292
1293 if (!sym)
1294 sym = reloc->sym;
1295
1296 if (sym->static_call_tramp) {
1297 list_add_tail(&insn->call_node, &file->static_call_list);
1298 return 0;
1299 }
1300
1301 if (sym->retpoline_thunk) {
1302 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1303 return 0;
1304 }
1305
1306 /*
1307 * Many compilers cannot disable KCOV or sanitizer calls with a function
1308 * attribute so they need a little help, NOP out any such calls from
1309 * noinstr text.
1310 */
1311 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1312 if (reloc)
1313 set_reloc_type(file->elf, reloc, R_NONE);
1314
1315 if (elf_write_insn(file->elf, insn->sec,
1316 insn->offset, insn->len,
1317 sibling ? arch_ret_insn(insn->len)
1318 : arch_nop_insn(insn->len))) {
1319 return -1;
1320 }
1321
1322 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1323
1324 if (sibling) {
1325 /*
1326 * We've replaced the tail-call JMP insn by two new
1327 * insn: RET; INT3, except we only have a single struct
1328 * insn here. Mark it retpoline_safe to avoid the SLS
1329 * warning, instead of adding another insn.
1330 */
1331 insn->retpoline_safe = true;
1332 }
1333
1334 return 0;
1335 }
1336
1337 if (opts.mcount && sym->fentry) {
1338 if (sibling)
1339 WARN_INSN(insn, "tail call to __fentry__ !?!?");
1340 if (opts.mnop) {
1341 if (reloc)
1342 set_reloc_type(file->elf, reloc, R_NONE);
1343
1344 if (elf_write_insn(file->elf, insn->sec,
1345 insn->offset, insn->len,
1346 arch_nop_insn(insn->len))) {
1347 return -1;
1348 }
1349
1350 insn->type = INSN_NOP;
1351 }
1352
1353 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1354 return 0;
1355 }
1356
1357 if (insn->type == INSN_CALL && !insn->sec->init &&
1358 !insn->_call_dest->embedded_insn)
1359 list_add_tail(&insn->call_node, &file->call_list);
1360
1361 if (!sibling && dead_end_function(file, sym))
1362 insn->dead_end = true;
1363
1364 return 0;
1365 }
1366
add_call_dest(struct objtool_file * file,struct instruction * insn,struct symbol * dest,bool sibling)1367 static int add_call_dest(struct objtool_file *file, struct instruction *insn,
1368 struct symbol *dest, bool sibling)
1369 {
1370 insn->_call_dest = dest;
1371 if (!dest)
1372 return 0;
1373
1374 /*
1375 * Whatever stack impact regular CALLs have, should be undone
1376 * by the RETURN of the called function.
1377 *
1378 * Annotated intra-function calls retain the stack_ops but
1379 * are converted to JUMP, see read_intra_function_calls().
1380 */
1381 remove_insn_ops(insn);
1382
1383 return annotate_call_site(file, insn, sibling);
1384 }
1385
add_retpoline_call(struct objtool_file * file,struct instruction * insn)1386 static int add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1387 {
1388 /*
1389 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1390 * so convert them accordingly.
1391 */
1392 switch (insn->type) {
1393 case INSN_CALL:
1394 insn->type = INSN_CALL_DYNAMIC;
1395 break;
1396 case INSN_JUMP_UNCONDITIONAL:
1397 insn->type = INSN_JUMP_DYNAMIC;
1398 break;
1399 case INSN_JUMP_CONDITIONAL:
1400 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1401 break;
1402 default:
1403 return 0;
1404 }
1405
1406 insn->retpoline_safe = true;
1407
1408 /*
1409 * Whatever stack impact regular CALLs have, should be undone
1410 * by the RETURN of the called function.
1411 *
1412 * Annotated intra-function calls retain the stack_ops but
1413 * are converted to JUMP, see read_intra_function_calls().
1414 */
1415 remove_insn_ops(insn);
1416
1417 return annotate_call_site(file, insn, false);
1418 }
1419
add_return_call(struct objtool_file * file,struct instruction * insn,bool add)1420 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1421 {
1422 /*
1423 * Return thunk tail calls are really just returns in disguise,
1424 * so convert them accordingly.
1425 */
1426 insn->type = INSN_RETURN;
1427 insn->retpoline_safe = true;
1428
1429 if (add)
1430 list_add_tail(&insn->call_node, &file->return_thunk_list);
1431 }
1432
is_first_func_insn(struct objtool_file * file,struct instruction * insn,struct symbol * sym)1433 static bool is_first_func_insn(struct objtool_file *file,
1434 struct instruction *insn, struct symbol *sym)
1435 {
1436 if (insn->offset == sym->offset)
1437 return true;
1438
1439 /* Allow direct CALL/JMP past ENDBR */
1440 if (opts.ibt) {
1441 struct instruction *prev = prev_insn_same_sym(file, insn);
1442
1443 if (prev && prev->type == INSN_ENDBR &&
1444 insn->offset == sym->offset + prev->len)
1445 return true;
1446 }
1447
1448 return false;
1449 }
1450
1451 /*
1452 * A sibling call is a tail-call to another symbol -- to differentiate from a
1453 * recursive tail-call which is to the same symbol.
1454 */
jump_is_sibling_call(struct objtool_file * file,struct instruction * from,struct instruction * to)1455 static bool jump_is_sibling_call(struct objtool_file *file,
1456 struct instruction *from, struct instruction *to)
1457 {
1458 struct symbol *fs = from->sym;
1459 struct symbol *ts = to->sym;
1460
1461 /* Not a sibling call if from/to a symbol hole */
1462 if (!fs || !ts)
1463 return false;
1464
1465 /* Not a sibling call if not targeting the start of a symbol. */
1466 if (!is_first_func_insn(file, to, ts))
1467 return false;
1468
1469 /* Disallow sibling calls into STT_NOTYPE */
1470 if (ts->type == STT_NOTYPE)
1471 return false;
1472
1473 /* Must not be self to be a sibling */
1474 return fs->pfunc != ts->pfunc;
1475 }
1476
1477 /*
1478 * Find the destination instructions for all jumps.
1479 */
add_jump_destinations(struct objtool_file * file)1480 static int add_jump_destinations(struct objtool_file *file)
1481 {
1482 struct instruction *insn, *jump_dest;
1483 struct reloc *reloc;
1484 struct section *dest_sec;
1485 unsigned long dest_off;
1486 int ret;
1487
1488 for_each_insn(file, insn) {
1489 struct symbol *func = insn_func(insn);
1490
1491 if (insn->jump_dest) {
1492 /*
1493 * handle_group_alt() may have previously set
1494 * 'jump_dest' for some alternatives.
1495 */
1496 continue;
1497 }
1498 if (!is_static_jump(insn))
1499 continue;
1500
1501 reloc = insn_reloc(file, insn);
1502 if (!reloc) {
1503 dest_sec = insn->sec;
1504 dest_off = arch_jump_destination(insn);
1505 } else if (reloc->sym->type == STT_SECTION) {
1506 dest_sec = reloc->sym->sec;
1507 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1508 } else if (reloc->sym->retpoline_thunk) {
1509 ret = add_retpoline_call(file, insn);
1510 if (ret)
1511 return ret;
1512 continue;
1513 } else if (reloc->sym->return_thunk) {
1514 add_return_call(file, insn, true);
1515 continue;
1516 } else if (func) {
1517 /*
1518 * External sibling call or internal sibling call with
1519 * STT_FUNC reloc.
1520 */
1521 ret = add_call_dest(file, insn, reloc->sym, true);
1522 if (ret)
1523 return ret;
1524 continue;
1525 } else if (reloc->sym->sec->idx) {
1526 dest_sec = reloc->sym->sec;
1527 dest_off = reloc->sym->sym.st_value +
1528 arch_dest_reloc_offset(reloc_addend(reloc));
1529 } else {
1530 /* non-func asm code jumping to another file */
1531 continue;
1532 }
1533
1534 jump_dest = find_insn(file, dest_sec, dest_off);
1535 if (!jump_dest) {
1536 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1537
1538 /*
1539 * This is a special case for retbleed_untrain_ret().
1540 * It jumps to __x86_return_thunk(), but objtool
1541 * can't find the thunk's starting RET
1542 * instruction, because the RET is also in the
1543 * middle of another instruction. Objtool only
1544 * knows about the outer instruction.
1545 */
1546 if (sym && sym->embedded_insn) {
1547 add_return_call(file, insn, false);
1548 continue;
1549 }
1550
1551 /*
1552 * GCOV/KCOV dead code can jump to the end of the
1553 * function/section.
1554 */
1555 if (file->ignore_unreachables && func &&
1556 dest_sec == insn->sec &&
1557 dest_off == func->offset + func->len)
1558 continue;
1559
1560 ERROR_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1561 dest_sec->name, dest_off);
1562 return -1;
1563 }
1564
1565 /*
1566 * An intra-TU jump in retpoline.o might not have a relocation
1567 * for its jump dest, in which case the above
1568 * add_{retpoline,return}_call() didn't happen.
1569 */
1570 if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1571 if (jump_dest->sym->retpoline_thunk) {
1572 ret = add_retpoline_call(file, insn);
1573 if (ret)
1574 return ret;
1575 continue;
1576 }
1577 if (jump_dest->sym->return_thunk) {
1578 add_return_call(file, insn, true);
1579 continue;
1580 }
1581 }
1582
1583 /*
1584 * Cross-function jump.
1585 */
1586 if (func && insn_func(jump_dest) && func != insn_func(jump_dest)) {
1587
1588 /*
1589 * For GCC 8+, create parent/child links for any cold
1590 * subfunctions. This is _mostly_ redundant with a
1591 * similar initialization in read_symbols().
1592 *
1593 * If a function has aliases, we want the *first* such
1594 * function in the symbol table to be the subfunction's
1595 * parent. In that case we overwrite the
1596 * initialization done in read_symbols().
1597 *
1598 * However this code can't completely replace the
1599 * read_symbols() code because this doesn't detect the
1600 * case where the parent function's only reference to a
1601 * subfunction is through a jump table.
1602 */
1603 if (!strstr(func->name, ".cold") &&
1604 strstr(insn_func(jump_dest)->name, ".cold")) {
1605 func->cfunc = insn_func(jump_dest);
1606 insn_func(jump_dest)->pfunc = func;
1607 }
1608 }
1609
1610 if (jump_is_sibling_call(file, insn, jump_dest)) {
1611 /*
1612 * Internal sibling call without reloc or with
1613 * STT_SECTION reloc.
1614 */
1615 ret = add_call_dest(file, insn, insn_func(jump_dest), true);
1616 if (ret)
1617 return ret;
1618 continue;
1619 }
1620
1621 insn->jump_dest = jump_dest;
1622 }
1623
1624 return 0;
1625 }
1626
find_call_destination(struct section * sec,unsigned long offset)1627 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1628 {
1629 struct symbol *call_dest;
1630
1631 call_dest = find_func_by_offset(sec, offset);
1632 if (!call_dest)
1633 call_dest = find_symbol_by_offset(sec, offset);
1634
1635 return call_dest;
1636 }
1637
1638 /*
1639 * Find the destination instructions for all calls.
1640 */
add_call_destinations(struct objtool_file * file)1641 static int add_call_destinations(struct objtool_file *file)
1642 {
1643 struct instruction *insn;
1644 unsigned long dest_off;
1645 struct symbol *dest;
1646 struct reloc *reloc;
1647 int ret;
1648
1649 for_each_insn(file, insn) {
1650 struct symbol *func = insn_func(insn);
1651 if (insn->type != INSN_CALL)
1652 continue;
1653
1654 reloc = insn_reloc(file, insn);
1655 if (!reloc) {
1656 dest_off = arch_jump_destination(insn);
1657 dest = find_call_destination(insn->sec, dest_off);
1658
1659 ret = add_call_dest(file, insn, dest, false);
1660 if (ret)
1661 return ret;
1662
1663 if (func && func->ignore)
1664 continue;
1665
1666 if (!insn_call_dest(insn)) {
1667 ERROR_INSN(insn, "unannotated intra-function call");
1668 return -1;
1669 }
1670
1671 if (func && insn_call_dest(insn)->type != STT_FUNC) {
1672 ERROR_INSN(insn, "unsupported call to non-function");
1673 return -1;
1674 }
1675
1676 } else if (reloc->sym->type == STT_SECTION) {
1677 dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1678 dest = find_call_destination(reloc->sym->sec, dest_off);
1679 if (!dest) {
1680 ERROR_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1681 reloc->sym->sec->name, dest_off);
1682 return -1;
1683 }
1684
1685 ret = add_call_dest(file, insn, dest, false);
1686 if (ret)
1687 return ret;
1688
1689 } else if (reloc->sym->retpoline_thunk) {
1690 ret = add_retpoline_call(file, insn);
1691 if (ret)
1692 return ret;
1693
1694 } else {
1695 ret = add_call_dest(file, insn, reloc->sym, false);
1696 if (ret)
1697 return ret;
1698 }
1699 }
1700
1701 return 0;
1702 }
1703
1704 /*
1705 * The .alternatives section requires some extra special care over and above
1706 * other special sections because alternatives are patched in place.
1707 */
handle_group_alt(struct objtool_file * file,struct special_alt * special_alt,struct instruction * orig_insn,struct instruction ** new_insn)1708 static int handle_group_alt(struct objtool_file *file,
1709 struct special_alt *special_alt,
1710 struct instruction *orig_insn,
1711 struct instruction **new_insn)
1712 {
1713 struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1714 struct alt_group *orig_alt_group, *new_alt_group;
1715 unsigned long dest_off;
1716
1717 orig_alt_group = orig_insn->alt_group;
1718 if (!orig_alt_group) {
1719 struct instruction *last_orig_insn = NULL;
1720
1721 orig_alt_group = calloc(1, sizeof(*orig_alt_group));
1722 if (!orig_alt_group) {
1723 ERROR_GLIBC("calloc");
1724 return -1;
1725 }
1726 orig_alt_group->cfi = calloc(special_alt->orig_len,
1727 sizeof(struct cfi_state *));
1728 if (!orig_alt_group->cfi) {
1729 ERROR_GLIBC("calloc");
1730 return -1;
1731 }
1732
1733 insn = orig_insn;
1734 sec_for_each_insn_from(file, insn) {
1735 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1736 break;
1737
1738 insn->alt_group = orig_alt_group;
1739 last_orig_insn = insn;
1740 }
1741 orig_alt_group->orig_group = NULL;
1742 orig_alt_group->first_insn = orig_insn;
1743 orig_alt_group->last_insn = last_orig_insn;
1744 orig_alt_group->nop = NULL;
1745 orig_alt_group->ignore = orig_insn->ignore_alts;
1746 } else {
1747 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1748 orig_alt_group->first_insn->offset != special_alt->orig_len) {
1749 ERROR_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1750 orig_alt_group->last_insn->offset +
1751 orig_alt_group->last_insn->len -
1752 orig_alt_group->first_insn->offset,
1753 special_alt->orig_len);
1754 return -1;
1755 }
1756 }
1757
1758 new_alt_group = calloc(1, sizeof(*new_alt_group));
1759 if (!new_alt_group) {
1760 ERROR_GLIBC("calloc");
1761 return -1;
1762 }
1763
1764 if (special_alt->new_len < special_alt->orig_len) {
1765 /*
1766 * Insert a fake nop at the end to make the replacement
1767 * alt_group the same size as the original. This is needed to
1768 * allow propagate_alt_cfi() to do its magic. When the last
1769 * instruction affects the stack, the instruction after it (the
1770 * nop) will propagate the new state to the shared CFI array.
1771 */
1772 nop = calloc(1, sizeof(*nop));
1773 if (!nop) {
1774 ERROR_GLIBC("calloc");
1775 return -1;
1776 }
1777 memset(nop, 0, sizeof(*nop));
1778
1779 nop->sec = special_alt->new_sec;
1780 nop->offset = special_alt->new_off + special_alt->new_len;
1781 nop->len = special_alt->orig_len - special_alt->new_len;
1782 nop->type = INSN_NOP;
1783 nop->sym = orig_insn->sym;
1784 nop->alt_group = new_alt_group;
1785 }
1786
1787 if (!special_alt->new_len) {
1788 *new_insn = nop;
1789 goto end;
1790 }
1791
1792 insn = *new_insn;
1793 sec_for_each_insn_from(file, insn) {
1794 struct reloc *alt_reloc;
1795
1796 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1797 break;
1798
1799 last_new_insn = insn;
1800
1801 insn->sym = orig_insn->sym;
1802 insn->alt_group = new_alt_group;
1803
1804 /*
1805 * Since alternative replacement code is copy/pasted by the
1806 * kernel after applying relocations, generally such code can't
1807 * have relative-address relocation references to outside the
1808 * .altinstr_replacement section, unless the arch's
1809 * alternatives code can adjust the relative offsets
1810 * accordingly.
1811 */
1812 alt_reloc = insn_reloc(file, insn);
1813 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1814 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1815
1816 ERROR_INSN(insn, "unsupported relocation in alternatives section");
1817 return -1;
1818 }
1819
1820 if (!is_static_jump(insn))
1821 continue;
1822
1823 if (!insn->immediate)
1824 continue;
1825
1826 dest_off = arch_jump_destination(insn);
1827 if (dest_off == special_alt->new_off + special_alt->new_len) {
1828 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1829 if (!insn->jump_dest) {
1830 ERROR_INSN(insn, "can't find alternative jump destination");
1831 return -1;
1832 }
1833 }
1834 }
1835
1836 if (!last_new_insn) {
1837 ERROR_FUNC(special_alt->new_sec, special_alt->new_off,
1838 "can't find last new alternative instruction");
1839 return -1;
1840 }
1841
1842 end:
1843 new_alt_group->orig_group = orig_alt_group;
1844 new_alt_group->first_insn = *new_insn;
1845 new_alt_group->last_insn = last_new_insn;
1846 new_alt_group->nop = nop;
1847 new_alt_group->ignore = (*new_insn)->ignore_alts;
1848 new_alt_group->cfi = orig_alt_group->cfi;
1849 return 0;
1850 }
1851
1852 /*
1853 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1854 * If the original instruction is a jump, make the alt entry an effective nop
1855 * by just skipping the original instruction.
1856 */
handle_jump_alt(struct objtool_file * file,struct special_alt * special_alt,struct instruction * orig_insn,struct instruction ** new_insn)1857 static int handle_jump_alt(struct objtool_file *file,
1858 struct special_alt *special_alt,
1859 struct instruction *orig_insn,
1860 struct instruction **new_insn)
1861 {
1862 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1863 orig_insn->type != INSN_NOP) {
1864
1865 ERROR_INSN(orig_insn, "unsupported instruction at jump label");
1866 return -1;
1867 }
1868
1869 if (opts.hack_jump_label && special_alt->key_addend & 2) {
1870 struct reloc *reloc = insn_reloc(file, orig_insn);
1871
1872 if (reloc)
1873 set_reloc_type(file->elf, reloc, R_NONE);
1874
1875 if (elf_write_insn(file->elf, orig_insn->sec,
1876 orig_insn->offset, orig_insn->len,
1877 arch_nop_insn(orig_insn->len))) {
1878 return -1;
1879 }
1880
1881 orig_insn->type = INSN_NOP;
1882 }
1883
1884 if (orig_insn->type == INSN_NOP) {
1885 if (orig_insn->len == 2)
1886 file->jl_nop_short++;
1887 else
1888 file->jl_nop_long++;
1889
1890 return 0;
1891 }
1892
1893 if (orig_insn->len == 2)
1894 file->jl_short++;
1895 else
1896 file->jl_long++;
1897
1898 *new_insn = next_insn_same_sec(file, orig_insn);
1899 return 0;
1900 }
1901
1902 /*
1903 * Read all the special sections which have alternate instructions which can be
1904 * patched in or redirected to at runtime. Each instruction having alternate
1905 * instruction(s) has them added to its insn->alts list, which will be
1906 * traversed in validate_branch().
1907 */
add_special_section_alts(struct objtool_file * file)1908 static int add_special_section_alts(struct objtool_file *file)
1909 {
1910 struct list_head special_alts;
1911 struct instruction *orig_insn, *new_insn;
1912 struct special_alt *special_alt, *tmp;
1913 struct alternative *alt;
1914 int ret;
1915
1916 if (special_get_alts(file->elf, &special_alts))
1917 return -1;
1918
1919 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1920
1921 orig_insn = find_insn(file, special_alt->orig_sec,
1922 special_alt->orig_off);
1923 if (!orig_insn) {
1924 ERROR_FUNC(special_alt->orig_sec, special_alt->orig_off,
1925 "special: can't find orig instruction");
1926 return -1;
1927 }
1928
1929 new_insn = NULL;
1930 if (!special_alt->group || special_alt->new_len) {
1931 new_insn = find_insn(file, special_alt->new_sec,
1932 special_alt->new_off);
1933 if (!new_insn) {
1934 ERROR_FUNC(special_alt->new_sec, special_alt->new_off,
1935 "special: can't find new instruction");
1936 return -1;
1937 }
1938 }
1939
1940 if (special_alt->group) {
1941 if (!special_alt->orig_len) {
1942 ERROR_INSN(orig_insn, "empty alternative entry");
1943 continue;
1944 }
1945
1946 ret = handle_group_alt(file, special_alt, orig_insn,
1947 &new_insn);
1948 if (ret)
1949 return ret;
1950
1951 } else if (special_alt->jump_or_nop) {
1952 ret = handle_jump_alt(file, special_alt, orig_insn,
1953 &new_insn);
1954 if (ret)
1955 return ret;
1956 }
1957
1958 alt = calloc(1, sizeof(*alt));
1959 if (!alt) {
1960 ERROR_GLIBC("calloc");
1961 return -1;
1962 }
1963
1964 alt->insn = new_insn;
1965 alt->next = orig_insn->alts;
1966 orig_insn->alts = alt;
1967
1968 list_del(&special_alt->list);
1969 free(special_alt);
1970 }
1971
1972 if (opts.stats) {
1973 printf("jl\\\tNOP\tJMP\n");
1974 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1975 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1976 }
1977
1978 return 0;
1979 }
1980
arch_jump_table_sym_offset(struct reloc * reloc,struct reloc * table)1981 __weak unsigned long arch_jump_table_sym_offset(struct reloc *reloc, struct reloc *table)
1982 {
1983 return reloc->sym->offset + reloc_addend(reloc);
1984 }
1985
add_jump_table(struct objtool_file * file,struct instruction * insn)1986 static int add_jump_table(struct objtool_file *file, struct instruction *insn)
1987 {
1988 unsigned long table_size = insn_jump_table_size(insn);
1989 struct symbol *pfunc = insn_func(insn)->pfunc;
1990 struct reloc *table = insn_jump_table(insn);
1991 struct instruction *dest_insn;
1992 unsigned int prev_offset = 0;
1993 struct reloc *reloc = table;
1994 struct alternative *alt;
1995 unsigned long sym_offset;
1996
1997 /*
1998 * Each @reloc is a switch table relocation which points to the target
1999 * instruction.
2000 */
2001 for_each_reloc_from(table->sec, reloc) {
2002
2003 /* Check for the end of the table: */
2004 if (table_size && reloc_offset(reloc) - reloc_offset(table) >= table_size)
2005 break;
2006 if (reloc != table && is_jump_table(reloc))
2007 break;
2008
2009 /* Make sure the table entries are consecutive: */
2010 if (prev_offset && reloc_offset(reloc) != prev_offset + arch_reloc_size(reloc))
2011 break;
2012
2013 sym_offset = arch_jump_table_sym_offset(reloc, table);
2014
2015 /* Detect function pointers from contiguous objects: */
2016 if (reloc->sym->sec == pfunc->sec && sym_offset == pfunc->offset)
2017 break;
2018
2019 /*
2020 * Clang sometimes leaves dangling unused jump table entries
2021 * which point to the end of the function. Ignore them.
2022 */
2023 if (reloc->sym->sec == pfunc->sec &&
2024 sym_offset == pfunc->offset + pfunc->len)
2025 goto next;
2026
2027 dest_insn = find_insn(file, reloc->sym->sec, sym_offset);
2028 if (!dest_insn)
2029 break;
2030
2031 /* Make sure the destination is in the same function: */
2032 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2033 break;
2034
2035 alt = calloc(1, sizeof(*alt));
2036 if (!alt) {
2037 ERROR_GLIBC("calloc");
2038 return -1;
2039 }
2040
2041 alt->insn = dest_insn;
2042 alt->next = insn->alts;
2043 insn->alts = alt;
2044 next:
2045 prev_offset = reloc_offset(reloc);
2046 }
2047
2048 if (!prev_offset) {
2049 ERROR_INSN(insn, "can't find switch jump table");
2050 return -1;
2051 }
2052
2053 return 0;
2054 }
2055
2056 /*
2057 * find_jump_table() - Given a dynamic jump, find the switch jump table
2058 * associated with it.
2059 */
find_jump_table(struct objtool_file * file,struct symbol * func,struct instruction * insn)2060 static void find_jump_table(struct objtool_file *file, struct symbol *func,
2061 struct instruction *insn)
2062 {
2063 struct reloc *table_reloc;
2064 struct instruction *dest_insn, *orig_insn = insn;
2065 unsigned long table_size;
2066 unsigned long sym_offset;
2067
2068 /*
2069 * Backward search using the @first_jump_src links, these help avoid
2070 * much of the 'in between' code. Which avoids us getting confused by
2071 * it.
2072 */
2073 for (;
2074 insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2075 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2076
2077 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2078 break;
2079
2080 /* allow small jumps within the range */
2081 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2082 insn->jump_dest &&
2083 (insn->jump_dest->offset <= insn->offset ||
2084 insn->jump_dest->offset > orig_insn->offset))
2085 break;
2086
2087 table_reloc = arch_find_switch_table(file, insn, &table_size);
2088 if (!table_reloc)
2089 continue;
2090
2091 sym_offset = table_reloc->sym->offset + reloc_addend(table_reloc);
2092
2093 dest_insn = find_insn(file, table_reloc->sym->sec, sym_offset);
2094 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2095 continue;
2096
2097 set_jump_table(table_reloc);
2098 orig_insn->_jump_table = table_reloc;
2099 orig_insn->_jump_table_size = table_size;
2100
2101 break;
2102 }
2103 }
2104
2105 /*
2106 * First pass: Mark the head of each jump table so that in the next pass,
2107 * we know when a given jump table ends and the next one starts.
2108 */
mark_func_jump_tables(struct objtool_file * file,struct symbol * func)2109 static void mark_func_jump_tables(struct objtool_file *file,
2110 struct symbol *func)
2111 {
2112 struct instruction *insn, *last = NULL;
2113
2114 func_for_each_insn(file, func, insn) {
2115 if (!last)
2116 last = insn;
2117
2118 /*
2119 * Store back-pointers for unconditional forward jumps such
2120 * that find_jump_table() can back-track using those and
2121 * avoid some potentially confusing code.
2122 */
2123 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2124 insn->offset > last->offset &&
2125 insn->jump_dest->offset > insn->offset &&
2126 !insn->jump_dest->first_jump_src) {
2127
2128 insn->jump_dest->first_jump_src = insn;
2129 last = insn->jump_dest;
2130 }
2131
2132 if (insn->type != INSN_JUMP_DYNAMIC)
2133 continue;
2134
2135 find_jump_table(file, func, insn);
2136 }
2137 }
2138
add_func_jump_tables(struct objtool_file * file,struct symbol * func)2139 static int add_func_jump_tables(struct objtool_file *file,
2140 struct symbol *func)
2141 {
2142 struct instruction *insn;
2143 int ret;
2144
2145 func_for_each_insn(file, func, insn) {
2146 if (!insn_jump_table(insn))
2147 continue;
2148
2149 ret = add_jump_table(file, insn);
2150 if (ret)
2151 return ret;
2152 }
2153
2154 return 0;
2155 }
2156
2157 /*
2158 * For some switch statements, gcc generates a jump table in the .rodata
2159 * section which contains a list of addresses within the function to jump to.
2160 * This finds these jump tables and adds them to the insn->alts lists.
2161 */
add_jump_table_alts(struct objtool_file * file)2162 static int add_jump_table_alts(struct objtool_file *file)
2163 {
2164 struct symbol *func;
2165 int ret;
2166
2167 if (!file->rodata)
2168 return 0;
2169
2170 for_each_sym(file, func) {
2171 if (func->type != STT_FUNC)
2172 continue;
2173
2174 mark_func_jump_tables(file, func);
2175 ret = add_func_jump_tables(file, func);
2176 if (ret)
2177 return ret;
2178 }
2179
2180 return 0;
2181 }
2182
set_func_state(struct cfi_state * state)2183 static void set_func_state(struct cfi_state *state)
2184 {
2185 state->cfa = initial_func_cfi.cfa;
2186 memcpy(&state->regs, &initial_func_cfi.regs,
2187 CFI_NUM_REGS * sizeof(struct cfi_reg));
2188 state->stack_size = initial_func_cfi.cfa.offset;
2189 state->type = UNWIND_HINT_TYPE_CALL;
2190 }
2191
read_unwind_hints(struct objtool_file * file)2192 static int read_unwind_hints(struct objtool_file *file)
2193 {
2194 struct cfi_state cfi = init_cfi;
2195 struct section *sec;
2196 struct unwind_hint *hint;
2197 struct instruction *insn;
2198 struct reloc *reloc;
2199 unsigned long offset;
2200 int i;
2201
2202 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2203 if (!sec)
2204 return 0;
2205
2206 if (!sec->rsec) {
2207 ERROR("missing .rela.discard.unwind_hints section");
2208 return -1;
2209 }
2210
2211 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2212 ERROR("struct unwind_hint size mismatch");
2213 return -1;
2214 }
2215
2216 file->hints = true;
2217
2218 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2219 hint = (struct unwind_hint *)sec->data->d_buf + i;
2220
2221 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2222 if (!reloc) {
2223 ERROR("can't find reloc for unwind_hints[%d]", i);
2224 return -1;
2225 }
2226
2227 if (reloc->sym->type == STT_SECTION) {
2228 offset = reloc_addend(reloc);
2229 } else if (reloc->sym->local_label) {
2230 offset = reloc->sym->offset;
2231 } else {
2232 ERROR("unexpected relocation symbol type in %s", sec->rsec->name);
2233 return -1;
2234 }
2235
2236 insn = find_insn(file, reloc->sym->sec, offset);
2237 if (!insn) {
2238 ERROR("can't find insn for unwind_hints[%d]", i);
2239 return -1;
2240 }
2241
2242 insn->hint = true;
2243
2244 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2245 insn->cfi = &force_undefined_cfi;
2246 continue;
2247 }
2248
2249 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2250 insn->hint = false;
2251 insn->save = true;
2252 continue;
2253 }
2254
2255 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2256 insn->restore = true;
2257 continue;
2258 }
2259
2260 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2261 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2262
2263 if (sym && sym->bind == STB_GLOBAL) {
2264 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2265 ERROR_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2266 return -1;
2267 }
2268 }
2269 }
2270
2271 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2272 insn->cfi = &func_cfi;
2273 continue;
2274 }
2275
2276 if (insn->cfi)
2277 cfi = *(insn->cfi);
2278
2279 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2280 ERROR_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2281 return -1;
2282 }
2283
2284 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2285 cfi.type = hint->type;
2286 cfi.signal = hint->signal;
2287
2288 insn->cfi = cfi_hash_find_or_add(&cfi);
2289 }
2290
2291 return 0;
2292 }
2293
read_annotate(struct objtool_file * file,int (* func)(struct objtool_file * file,int type,struct instruction * insn))2294 static int read_annotate(struct objtool_file *file,
2295 int (*func)(struct objtool_file *file, int type, struct instruction *insn))
2296 {
2297 struct section *sec;
2298 struct instruction *insn;
2299 struct reloc *reloc;
2300 uint64_t offset;
2301 int type, ret;
2302
2303 sec = find_section_by_name(file->elf, ".discard.annotate_insn");
2304 if (!sec)
2305 return 0;
2306
2307 if (!sec->rsec)
2308 return 0;
2309
2310 if (sec->sh.sh_entsize != 8) {
2311 static bool warned = false;
2312 if (!warned && opts.verbose) {
2313 WARN("%s: dodgy linker, sh_entsize != 8", sec->name);
2314 warned = true;
2315 }
2316 sec->sh.sh_entsize = 8;
2317 }
2318
2319 for_each_reloc(sec->rsec, reloc) {
2320 type = *(u32 *)(sec->data->d_buf + (reloc_idx(reloc) * sec->sh.sh_entsize) + 4);
2321
2322 offset = reloc->sym->offset + reloc_addend(reloc);
2323 insn = find_insn(file, reloc->sym->sec, offset);
2324
2325 if (!insn) {
2326 ERROR("bad .discard.annotate_insn entry: %d of type %d", reloc_idx(reloc), type);
2327 return -1;
2328 }
2329
2330 ret = func(file, type, insn);
2331 if (ret < 0)
2332 return ret;
2333 }
2334
2335 return 0;
2336 }
2337
__annotate_early(struct objtool_file * file,int type,struct instruction * insn)2338 static int __annotate_early(struct objtool_file *file, int type, struct instruction *insn)
2339 {
2340 switch (type) {
2341
2342 /* Must be before add_special_section_alts() */
2343 case ANNOTYPE_IGNORE_ALTS:
2344 insn->ignore_alts = true;
2345 break;
2346
2347 /*
2348 * Must be before read_unwind_hints() since that needs insn->noendbr.
2349 */
2350 case ANNOTYPE_NOENDBR:
2351 insn->noendbr = 1;
2352 break;
2353
2354 default:
2355 break;
2356 }
2357
2358 return 0;
2359 }
2360
__annotate_ifc(struct objtool_file * file,int type,struct instruction * insn)2361 static int __annotate_ifc(struct objtool_file *file, int type, struct instruction *insn)
2362 {
2363 unsigned long dest_off;
2364
2365 if (type != ANNOTYPE_INTRA_FUNCTION_CALL)
2366 return 0;
2367
2368 if (insn->type != INSN_CALL) {
2369 ERROR_INSN(insn, "intra_function_call not a direct call");
2370 return -1;
2371 }
2372
2373 /*
2374 * Treat intra-function CALLs as JMPs, but with a stack_op.
2375 * See add_call_destinations(), which strips stack_ops from
2376 * normal CALLs.
2377 */
2378 insn->type = INSN_JUMP_UNCONDITIONAL;
2379
2380 dest_off = arch_jump_destination(insn);
2381 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2382 if (!insn->jump_dest) {
2383 ERROR_INSN(insn, "can't find call dest at %s+0x%lx",
2384 insn->sec->name, dest_off);
2385 return -1;
2386 }
2387
2388 return 0;
2389 }
2390
__annotate_late(struct objtool_file * file,int type,struct instruction * insn)2391 static int __annotate_late(struct objtool_file *file, int type, struct instruction *insn)
2392 {
2393 switch (type) {
2394 case ANNOTYPE_NOENDBR:
2395 /* early */
2396 break;
2397
2398 case ANNOTYPE_RETPOLINE_SAFE:
2399 if (insn->type != INSN_JUMP_DYNAMIC &&
2400 insn->type != INSN_CALL_DYNAMIC &&
2401 insn->type != INSN_RETURN &&
2402 insn->type != INSN_NOP) {
2403 ERROR_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2404 return -1;
2405 }
2406
2407 insn->retpoline_safe = true;
2408 break;
2409
2410 case ANNOTYPE_INSTR_BEGIN:
2411 insn->instr++;
2412 break;
2413
2414 case ANNOTYPE_INSTR_END:
2415 insn->instr--;
2416 break;
2417
2418 case ANNOTYPE_UNRET_BEGIN:
2419 insn->unret = 1;
2420 break;
2421
2422 case ANNOTYPE_IGNORE_ALTS:
2423 /* early */
2424 break;
2425
2426 case ANNOTYPE_INTRA_FUNCTION_CALL:
2427 /* ifc */
2428 break;
2429
2430 case ANNOTYPE_REACHABLE:
2431 insn->dead_end = false;
2432 break;
2433
2434 default:
2435 ERROR_INSN(insn, "Unknown annotation type: %d", type);
2436 return -1;
2437 }
2438
2439 return 0;
2440 }
2441
2442 /*
2443 * Return true if name matches an instrumentation function, where calls to that
2444 * function from noinstr code can safely be removed, but compilers won't do so.
2445 */
is_profiling_func(const char * name)2446 static bool is_profiling_func(const char *name)
2447 {
2448 /*
2449 * Many compilers cannot disable KCOV with a function attribute.
2450 */
2451 if (!strncmp(name, "__sanitizer_cov_", 16))
2452 return true;
2453
2454 /*
2455 * Some compilers currently do not remove __tsan_func_entry/exit nor
2456 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2457 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2458 * minimum Clang version is 14.0, this can be removed.
2459 */
2460 if (!strncmp(name, "__tsan_func_", 12) ||
2461 !strcmp(name, "__tsan_atomic_signal_fence"))
2462 return true;
2463
2464 return false;
2465 }
2466
classify_symbols(struct objtool_file * file)2467 static int classify_symbols(struct objtool_file *file)
2468 {
2469 struct symbol *func;
2470
2471 for_each_sym(file, func) {
2472 if (func->type == STT_NOTYPE && strstarts(func->name, ".L"))
2473 func->local_label = true;
2474
2475 if (func->bind != STB_GLOBAL)
2476 continue;
2477
2478 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2479 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2480 func->static_call_tramp = true;
2481
2482 if (arch_is_retpoline(func))
2483 func->retpoline_thunk = true;
2484
2485 if (arch_is_rethunk(func))
2486 func->return_thunk = true;
2487
2488 if (arch_is_embedded_insn(func))
2489 func->embedded_insn = true;
2490
2491 if (arch_ftrace_match(func->name))
2492 func->fentry = true;
2493
2494 if (is_profiling_func(func->name))
2495 func->profiling_func = true;
2496 }
2497
2498 return 0;
2499 }
2500
mark_rodata(struct objtool_file * file)2501 static void mark_rodata(struct objtool_file *file)
2502 {
2503 struct section *sec;
2504 bool found = false;
2505
2506 /*
2507 * Search for the following rodata sections, each of which can
2508 * potentially contain jump tables:
2509 *
2510 * - .rodata: can contain GCC switch tables
2511 * - .rodata.<func>: same, if -fdata-sections is being used
2512 * - .data.rel.ro.c_jump_table: contains C annotated jump tables
2513 *
2514 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2515 */
2516 for_each_sec(file, sec) {
2517 if ((!strncmp(sec->name, ".rodata", 7) &&
2518 !strstr(sec->name, ".str1.")) ||
2519 !strncmp(sec->name, ".data.rel.ro", 12)) {
2520 sec->rodata = true;
2521 found = true;
2522 }
2523 }
2524
2525 file->rodata = found;
2526 }
2527
decode_sections(struct objtool_file * file)2528 static int decode_sections(struct objtool_file *file)
2529 {
2530 int ret;
2531
2532 mark_rodata(file);
2533
2534 ret = init_pv_ops(file);
2535 if (ret)
2536 return ret;
2537
2538 /*
2539 * Must be before add_{jump_call}_destination.
2540 */
2541 ret = classify_symbols(file);
2542 if (ret)
2543 return ret;
2544
2545 ret = decode_instructions(file);
2546 if (ret)
2547 return ret;
2548
2549 ret = add_ignores(file);
2550 if (ret)
2551 return ret;
2552
2553 add_uaccess_safe(file);
2554
2555 ret = read_annotate(file, __annotate_early);
2556 if (ret)
2557 return ret;
2558
2559 /*
2560 * Must be before add_jump_destinations(), which depends on 'func'
2561 * being set for alternatives, to enable proper sibling call detection.
2562 */
2563 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2564 ret = add_special_section_alts(file);
2565 if (ret)
2566 return ret;
2567 }
2568
2569 ret = add_jump_destinations(file);
2570 if (ret)
2571 return ret;
2572
2573 /*
2574 * Must be before add_call_destination(); it changes INSN_CALL to
2575 * INSN_JUMP.
2576 */
2577 ret = read_annotate(file, __annotate_ifc);
2578 if (ret)
2579 return ret;
2580
2581 ret = add_call_destinations(file);
2582 if (ret)
2583 return ret;
2584
2585 ret = add_jump_table_alts(file);
2586 if (ret)
2587 return ret;
2588
2589 ret = read_unwind_hints(file);
2590 if (ret)
2591 return ret;
2592
2593 /*
2594 * Must be after add_call_destinations() such that it can override
2595 * dead_end_function() marks.
2596 */
2597 ret = read_annotate(file, __annotate_late);
2598 if (ret)
2599 return ret;
2600
2601 return 0;
2602 }
2603
is_special_call(struct instruction * insn)2604 static bool is_special_call(struct instruction *insn)
2605 {
2606 if (insn->type == INSN_CALL) {
2607 struct symbol *dest = insn_call_dest(insn);
2608
2609 if (!dest)
2610 return false;
2611
2612 if (dest->fentry || dest->embedded_insn)
2613 return true;
2614 }
2615
2616 return false;
2617 }
2618
has_modified_stack_frame(struct instruction * insn,struct insn_state * state)2619 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2620 {
2621 struct cfi_state *cfi = &state->cfi;
2622 int i;
2623
2624 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2625 return true;
2626
2627 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2628 return true;
2629
2630 if (cfi->stack_size != initial_func_cfi.cfa.offset)
2631 return true;
2632
2633 for (i = 0; i < CFI_NUM_REGS; i++) {
2634 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2635 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2636 return true;
2637 }
2638
2639 return false;
2640 }
2641
check_reg_frame_pos(const struct cfi_reg * reg,int expected_offset)2642 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2643 int expected_offset)
2644 {
2645 return reg->base == CFI_CFA &&
2646 reg->offset == expected_offset;
2647 }
2648
has_valid_stack_frame(struct insn_state * state)2649 static bool has_valid_stack_frame(struct insn_state *state)
2650 {
2651 struct cfi_state *cfi = &state->cfi;
2652
2653 if (cfi->cfa.base == CFI_BP &&
2654 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2655 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2656 return true;
2657
2658 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2659 return true;
2660
2661 return false;
2662 }
2663
update_cfi_state_regs(struct instruction * insn,struct cfi_state * cfi,struct stack_op * op)2664 static int update_cfi_state_regs(struct instruction *insn,
2665 struct cfi_state *cfi,
2666 struct stack_op *op)
2667 {
2668 struct cfi_reg *cfa = &cfi->cfa;
2669
2670 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2671 return 0;
2672
2673 /* push */
2674 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2675 cfa->offset += 8;
2676
2677 /* pop */
2678 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2679 cfa->offset -= 8;
2680
2681 /* add immediate to sp */
2682 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2683 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2684 cfa->offset -= op->src.offset;
2685
2686 return 0;
2687 }
2688
save_reg(struct cfi_state * cfi,unsigned char reg,int base,int offset)2689 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2690 {
2691 if (arch_callee_saved_reg(reg) &&
2692 cfi->regs[reg].base == CFI_UNDEFINED) {
2693 cfi->regs[reg].base = base;
2694 cfi->regs[reg].offset = offset;
2695 }
2696 }
2697
restore_reg(struct cfi_state * cfi,unsigned char reg)2698 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2699 {
2700 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2701 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2702 }
2703
2704 /*
2705 * A note about DRAP stack alignment:
2706 *
2707 * GCC has the concept of a DRAP register, which is used to help keep track of
2708 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2709 * register. The typical DRAP pattern is:
2710 *
2711 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2712 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2713 * 41 ff 72 f8 pushq -0x8(%r10)
2714 * 55 push %rbp
2715 * 48 89 e5 mov %rsp,%rbp
2716 * (more pushes)
2717 * 41 52 push %r10
2718 * ...
2719 * 41 5a pop %r10
2720 * (more pops)
2721 * 5d pop %rbp
2722 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2723 * c3 retq
2724 *
2725 * There are some variations in the epilogues, like:
2726 *
2727 * 5b pop %rbx
2728 * 41 5a pop %r10
2729 * 41 5c pop %r12
2730 * 41 5d pop %r13
2731 * 41 5e pop %r14
2732 * c9 leaveq
2733 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2734 * c3 retq
2735 *
2736 * and:
2737 *
2738 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2739 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2740 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2741 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2742 * c9 leaveq
2743 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2744 * c3 retq
2745 *
2746 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2747 * restored beforehand:
2748 *
2749 * 41 55 push %r13
2750 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2751 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2752 * ...
2753 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2754 * 41 5d pop %r13
2755 * c3 retq
2756 */
update_cfi_state(struct instruction * insn,struct instruction * next_insn,struct cfi_state * cfi,struct stack_op * op)2757 static int update_cfi_state(struct instruction *insn,
2758 struct instruction *next_insn,
2759 struct cfi_state *cfi, struct stack_op *op)
2760 {
2761 struct cfi_reg *cfa = &cfi->cfa;
2762 struct cfi_reg *regs = cfi->regs;
2763
2764 /* ignore UNWIND_HINT_UNDEFINED regions */
2765 if (cfi->force_undefined)
2766 return 0;
2767
2768 /* stack operations don't make sense with an undefined CFA */
2769 if (cfa->base == CFI_UNDEFINED) {
2770 if (insn_func(insn)) {
2771 WARN_INSN(insn, "undefined stack state");
2772 return 1;
2773 }
2774 return 0;
2775 }
2776
2777 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2778 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2779 return update_cfi_state_regs(insn, cfi, op);
2780
2781 switch (op->dest.type) {
2782
2783 case OP_DEST_REG:
2784 switch (op->src.type) {
2785
2786 case OP_SRC_REG:
2787 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2788 cfa->base == CFI_SP &&
2789 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) {
2790
2791 /* mov %rsp, %rbp */
2792 cfa->base = op->dest.reg;
2793 cfi->bp_scratch = false;
2794 }
2795
2796 else if (op->src.reg == CFI_SP &&
2797 op->dest.reg == CFI_BP && cfi->drap) {
2798
2799 /* drap: mov %rsp, %rbp */
2800 regs[CFI_BP].base = CFI_BP;
2801 regs[CFI_BP].offset = -cfi->stack_size;
2802 cfi->bp_scratch = false;
2803 }
2804
2805 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2806
2807 /*
2808 * mov %rsp, %reg
2809 *
2810 * This is needed for the rare case where GCC
2811 * does:
2812 *
2813 * mov %rsp, %rax
2814 * ...
2815 * mov %rax, %rsp
2816 */
2817 cfi->vals[op->dest.reg].base = CFI_CFA;
2818 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2819 }
2820
2821 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2822 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2823
2824 /*
2825 * mov %rbp, %rsp
2826 *
2827 * Restore the original stack pointer (Clang).
2828 */
2829 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2830 }
2831
2832 else if (op->dest.reg == cfa->base) {
2833
2834 /* mov %reg, %rsp */
2835 if (cfa->base == CFI_SP &&
2836 cfi->vals[op->src.reg].base == CFI_CFA) {
2837
2838 /*
2839 * This is needed for the rare case
2840 * where GCC does something dumb like:
2841 *
2842 * lea 0x8(%rsp), %rcx
2843 * ...
2844 * mov %rcx, %rsp
2845 */
2846 cfa->offset = -cfi->vals[op->src.reg].offset;
2847 cfi->stack_size = cfa->offset;
2848
2849 } else if (cfa->base == CFI_SP &&
2850 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2851 cfi->vals[op->src.reg].offset == cfa->offset) {
2852
2853 /*
2854 * Stack swizzle:
2855 *
2856 * 1: mov %rsp, (%[tos])
2857 * 2: mov %[tos], %rsp
2858 * ...
2859 * 3: pop %rsp
2860 *
2861 * Where:
2862 *
2863 * 1 - places a pointer to the previous
2864 * stack at the Top-of-Stack of the
2865 * new stack.
2866 *
2867 * 2 - switches to the new stack.
2868 *
2869 * 3 - pops the Top-of-Stack to restore
2870 * the original stack.
2871 *
2872 * Note: we set base to SP_INDIRECT
2873 * here and preserve offset. Therefore
2874 * when the unwinder reaches ToS it
2875 * will dereference SP and then add the
2876 * offset to find the next frame, IOW:
2877 * (%rsp) + offset.
2878 */
2879 cfa->base = CFI_SP_INDIRECT;
2880
2881 } else {
2882 cfa->base = CFI_UNDEFINED;
2883 cfa->offset = 0;
2884 }
2885 }
2886
2887 else if (op->dest.reg == CFI_SP &&
2888 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2889 cfi->vals[op->src.reg].offset == cfa->offset) {
2890
2891 /*
2892 * The same stack swizzle case 2) as above. But
2893 * because we can't change cfa->base, case 3)
2894 * will become a regular POP. Pretend we're a
2895 * PUSH so things don't go unbalanced.
2896 */
2897 cfi->stack_size += 8;
2898 }
2899
2900
2901 break;
2902
2903 case OP_SRC_ADD:
2904 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2905
2906 /* add imm, %rsp */
2907 cfi->stack_size -= op->src.offset;
2908 if (cfa->base == CFI_SP)
2909 cfa->offset -= op->src.offset;
2910 break;
2911 }
2912
2913 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP &&
2914 insn->sym->frame_pointer) {
2915 /* addi.d fp,sp,imm on LoongArch */
2916 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) {
2917 cfa->base = CFI_BP;
2918 cfa->offset = 0;
2919 }
2920 break;
2921 }
2922
2923 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2924 /* addi.d sp,fp,imm on LoongArch */
2925 if (cfa->base == CFI_BP && cfa->offset == 0) {
2926 if (insn->sym->frame_pointer) {
2927 cfa->base = CFI_SP;
2928 cfa->offset = -op->src.offset;
2929 }
2930 } else {
2931 /* lea disp(%rbp), %rsp */
2932 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2933 }
2934 break;
2935 }
2936
2937 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2938
2939 /* drap: lea disp(%rsp), %drap */
2940 cfi->drap_reg = op->dest.reg;
2941
2942 /*
2943 * lea disp(%rsp), %reg
2944 *
2945 * This is needed for the rare case where GCC
2946 * does something dumb like:
2947 *
2948 * lea 0x8(%rsp), %rcx
2949 * ...
2950 * mov %rcx, %rsp
2951 */
2952 cfi->vals[op->dest.reg].base = CFI_CFA;
2953 cfi->vals[op->dest.reg].offset = \
2954 -cfi->stack_size + op->src.offset;
2955
2956 break;
2957 }
2958
2959 if (cfi->drap && op->dest.reg == CFI_SP &&
2960 op->src.reg == cfi->drap_reg) {
2961
2962 /* drap: lea disp(%drap), %rsp */
2963 cfa->base = CFI_SP;
2964 cfa->offset = cfi->stack_size = -op->src.offset;
2965 cfi->drap_reg = CFI_UNDEFINED;
2966 cfi->drap = false;
2967 break;
2968 }
2969
2970 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
2971 WARN_INSN(insn, "unsupported stack register modification");
2972 return -1;
2973 }
2974
2975 break;
2976
2977 case OP_SRC_AND:
2978 if (op->dest.reg != CFI_SP ||
2979 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2980 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
2981 WARN_INSN(insn, "unsupported stack pointer realignment");
2982 return -1;
2983 }
2984
2985 if (cfi->drap_reg != CFI_UNDEFINED) {
2986 /* drap: and imm, %rsp */
2987 cfa->base = cfi->drap_reg;
2988 cfa->offset = cfi->stack_size = 0;
2989 cfi->drap = true;
2990 }
2991
2992 /*
2993 * Older versions of GCC (4.8ish) realign the stack
2994 * without DRAP, with a frame pointer.
2995 */
2996
2997 break;
2998
2999 case OP_SRC_POP:
3000 case OP_SRC_POPF:
3001 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3002
3003 /* pop %rsp; # restore from a stack swizzle */
3004 cfa->base = CFI_SP;
3005 break;
3006 }
3007
3008 if (!cfi->drap && op->dest.reg == cfa->base) {
3009
3010 /* pop %rbp */
3011 cfa->base = CFI_SP;
3012 }
3013
3014 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3015 op->dest.reg == cfi->drap_reg &&
3016 cfi->drap_offset == -cfi->stack_size) {
3017
3018 /* drap: pop %drap */
3019 cfa->base = cfi->drap_reg;
3020 cfa->offset = 0;
3021 cfi->drap_offset = -1;
3022
3023 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3024
3025 /* pop %reg */
3026 restore_reg(cfi, op->dest.reg);
3027 }
3028
3029 cfi->stack_size -= 8;
3030 if (cfa->base == CFI_SP)
3031 cfa->offset -= 8;
3032
3033 break;
3034
3035 case OP_SRC_REG_INDIRECT:
3036 if (!cfi->drap && op->dest.reg == cfa->base &&
3037 op->dest.reg == CFI_BP) {
3038
3039 /* mov disp(%rsp), %rbp */
3040 cfa->base = CFI_SP;
3041 cfa->offset = cfi->stack_size;
3042 }
3043
3044 if (cfi->drap && op->src.reg == CFI_BP &&
3045 op->src.offset == cfi->drap_offset) {
3046
3047 /* drap: mov disp(%rbp), %drap */
3048 cfa->base = cfi->drap_reg;
3049 cfa->offset = 0;
3050 cfi->drap_offset = -1;
3051 }
3052
3053 if (cfi->drap && op->src.reg == CFI_BP &&
3054 op->src.offset == regs[op->dest.reg].offset) {
3055
3056 /* drap: mov disp(%rbp), %reg */
3057 restore_reg(cfi, op->dest.reg);
3058
3059 } else if (op->src.reg == cfa->base &&
3060 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3061
3062 /* mov disp(%rbp), %reg */
3063 /* mov disp(%rsp), %reg */
3064 restore_reg(cfi, op->dest.reg);
3065
3066 } else if (op->src.reg == CFI_SP &&
3067 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3068
3069 /* mov disp(%rsp), %reg */
3070 restore_reg(cfi, op->dest.reg);
3071 }
3072
3073 break;
3074
3075 default:
3076 WARN_INSN(insn, "unknown stack-related instruction");
3077 return -1;
3078 }
3079
3080 break;
3081
3082 case OP_DEST_PUSH:
3083 case OP_DEST_PUSHF:
3084 cfi->stack_size += 8;
3085 if (cfa->base == CFI_SP)
3086 cfa->offset += 8;
3087
3088 if (op->src.type != OP_SRC_REG)
3089 break;
3090
3091 if (cfi->drap) {
3092 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3093
3094 /* drap: push %drap */
3095 cfa->base = CFI_BP_INDIRECT;
3096 cfa->offset = -cfi->stack_size;
3097
3098 /* save drap so we know when to restore it */
3099 cfi->drap_offset = -cfi->stack_size;
3100
3101 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3102
3103 /* drap: push %rbp */
3104 cfi->stack_size = 0;
3105
3106 } else {
3107
3108 /* drap: push %reg */
3109 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3110 }
3111
3112 } else {
3113
3114 /* push %reg */
3115 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3116 }
3117
3118 /* detect when asm code uses rbp as a scratch register */
3119 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3120 cfa->base != CFI_BP)
3121 cfi->bp_scratch = true;
3122 break;
3123
3124 case OP_DEST_REG_INDIRECT:
3125
3126 if (cfi->drap) {
3127 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3128
3129 /* drap: mov %drap, disp(%rbp) */
3130 cfa->base = CFI_BP_INDIRECT;
3131 cfa->offset = op->dest.offset;
3132
3133 /* save drap offset so we know when to restore it */
3134 cfi->drap_offset = op->dest.offset;
3135 } else {
3136
3137 /* drap: mov reg, disp(%rbp) */
3138 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3139 }
3140
3141 } else if (op->dest.reg == cfa->base) {
3142
3143 /* mov reg, disp(%rbp) */
3144 /* mov reg, disp(%rsp) */
3145 save_reg(cfi, op->src.reg, CFI_CFA,
3146 op->dest.offset - cfi->cfa.offset);
3147
3148 } else if (op->dest.reg == CFI_SP) {
3149
3150 /* mov reg, disp(%rsp) */
3151 save_reg(cfi, op->src.reg, CFI_CFA,
3152 op->dest.offset - cfi->stack_size);
3153
3154 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3155
3156 /* mov %rsp, (%reg); # setup a stack swizzle. */
3157 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3158 cfi->vals[op->dest.reg].offset = cfa->offset;
3159 }
3160
3161 break;
3162
3163 case OP_DEST_MEM:
3164 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3165 WARN_INSN(insn, "unknown stack-related memory operation");
3166 return -1;
3167 }
3168
3169 /* pop mem */
3170 cfi->stack_size -= 8;
3171 if (cfa->base == CFI_SP)
3172 cfa->offset -= 8;
3173
3174 break;
3175
3176 default:
3177 WARN_INSN(insn, "unknown stack-related instruction");
3178 return -1;
3179 }
3180
3181 return 0;
3182 }
3183
3184 /*
3185 * The stack layouts of alternatives instructions can sometimes diverge when
3186 * they have stack modifications. That's fine as long as the potential stack
3187 * layouts don't conflict at any given potential instruction boundary.
3188 *
3189 * Flatten the CFIs of the different alternative code streams (both original
3190 * and replacement) into a single shared CFI array which can be used to detect
3191 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3192 */
propagate_alt_cfi(struct objtool_file * file,struct instruction * insn)3193 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3194 {
3195 struct cfi_state **alt_cfi;
3196 int group_off;
3197
3198 if (!insn->alt_group)
3199 return 0;
3200
3201 if (!insn->cfi) {
3202 WARN("CFI missing");
3203 return -1;
3204 }
3205
3206 alt_cfi = insn->alt_group->cfi;
3207 group_off = insn->offset - insn->alt_group->first_insn->offset;
3208
3209 if (!alt_cfi[group_off]) {
3210 alt_cfi[group_off] = insn->cfi;
3211 } else {
3212 if (cficmp(alt_cfi[group_off], insn->cfi)) {
3213 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3214 struct instruction *orig = orig_group->first_insn;
3215 WARN_INSN(orig, "stack layout conflict in alternatives: %s",
3216 offstr(insn->sec, insn->offset));
3217 return -1;
3218 }
3219 }
3220
3221 return 0;
3222 }
3223
handle_insn_ops(struct instruction * insn,struct instruction * next_insn,struct insn_state * state)3224 static int handle_insn_ops(struct instruction *insn,
3225 struct instruction *next_insn,
3226 struct insn_state *state)
3227 {
3228 struct stack_op *op;
3229 int ret;
3230
3231 for (op = insn->stack_ops; op; op = op->next) {
3232
3233 ret = update_cfi_state(insn, next_insn, &state->cfi, op);
3234 if (ret)
3235 return ret;
3236
3237 if (!opts.uaccess || !insn->alt_group)
3238 continue;
3239
3240 if (op->dest.type == OP_DEST_PUSHF) {
3241 if (!state->uaccess_stack) {
3242 state->uaccess_stack = 1;
3243 } else if (state->uaccess_stack >> 31) {
3244 WARN_INSN(insn, "PUSHF stack exhausted");
3245 return 1;
3246 }
3247 state->uaccess_stack <<= 1;
3248 state->uaccess_stack |= state->uaccess;
3249 }
3250
3251 if (op->src.type == OP_SRC_POPF) {
3252 if (state->uaccess_stack) {
3253 state->uaccess = state->uaccess_stack & 1;
3254 state->uaccess_stack >>= 1;
3255 if (state->uaccess_stack == 1)
3256 state->uaccess_stack = 0;
3257 }
3258 }
3259 }
3260
3261 return 0;
3262 }
3263
insn_cfi_match(struct instruction * insn,struct cfi_state * cfi2)3264 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3265 {
3266 struct cfi_state *cfi1 = insn->cfi;
3267 int i;
3268
3269 if (!cfi1) {
3270 WARN("CFI missing");
3271 return false;
3272 }
3273
3274 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3275
3276 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3277 cfi1->cfa.base, cfi1->cfa.offset,
3278 cfi2->cfa.base, cfi2->cfa.offset);
3279 return false;
3280
3281 }
3282
3283 if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3284 for (i = 0; i < CFI_NUM_REGS; i++) {
3285
3286 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], sizeof(struct cfi_reg)))
3287 continue;
3288
3289 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3290 i, cfi1->regs[i].base, cfi1->regs[i].offset,
3291 i, cfi2->regs[i].base, cfi2->regs[i].offset);
3292 }
3293 return false;
3294 }
3295
3296 if (cfi1->type != cfi2->type) {
3297
3298 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3299 cfi1->type, cfi2->type);
3300 return false;
3301 }
3302
3303 if (cfi1->drap != cfi2->drap ||
3304 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3305 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3306
3307 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3308 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3309 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3310 return false;
3311 }
3312
3313 return true;
3314 }
3315
func_uaccess_safe(struct symbol * func)3316 static inline bool func_uaccess_safe(struct symbol *func)
3317 {
3318 if (func)
3319 return func->uaccess_safe;
3320
3321 return false;
3322 }
3323
call_dest_name(struct instruction * insn)3324 static inline const char *call_dest_name(struct instruction *insn)
3325 {
3326 static char pvname[19];
3327 struct reloc *reloc;
3328 int idx;
3329
3330 if (insn_call_dest(insn))
3331 return insn_call_dest(insn)->name;
3332
3333 reloc = insn_reloc(NULL, insn);
3334 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3335 idx = (reloc_addend(reloc) / sizeof(void *));
3336 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3337 return pvname;
3338 }
3339
3340 return "{dynamic}";
3341 }
3342
pv_call_dest(struct objtool_file * file,struct instruction * insn)3343 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3344 {
3345 struct symbol *target;
3346 struct reloc *reloc;
3347 int idx;
3348
3349 reloc = insn_reloc(file, insn);
3350 if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3351 return false;
3352
3353 idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3354
3355 if (file->pv_ops[idx].clean)
3356 return true;
3357
3358 file->pv_ops[idx].clean = true;
3359
3360 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3361 if (!target->sec->noinstr) {
3362 WARN("pv_ops[%d]: %s", idx, target->name);
3363 file->pv_ops[idx].clean = false;
3364 }
3365 }
3366
3367 return file->pv_ops[idx].clean;
3368 }
3369
noinstr_call_dest(struct objtool_file * file,struct instruction * insn,struct symbol * func)3370 static inline bool noinstr_call_dest(struct objtool_file *file,
3371 struct instruction *insn,
3372 struct symbol *func)
3373 {
3374 /*
3375 * We can't deal with indirect function calls at present;
3376 * assume they're instrumented.
3377 */
3378 if (!func) {
3379 if (file->pv_ops)
3380 return pv_call_dest(file, insn);
3381
3382 return false;
3383 }
3384
3385 /*
3386 * If the symbol is from a noinstr section; we good.
3387 */
3388 if (func->sec->noinstr)
3389 return true;
3390
3391 /*
3392 * If the symbol is a static_call trampoline, we can't tell.
3393 */
3394 if (func->static_call_tramp)
3395 return true;
3396
3397 /*
3398 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3399 * something 'BAD' happened. At the risk of taking the machine down,
3400 * let them proceed to get the message out.
3401 */
3402 if (!strncmp(func->name, "__ubsan_handle_", 15))
3403 return true;
3404
3405 return false;
3406 }
3407
validate_call(struct objtool_file * file,struct instruction * insn,struct insn_state * state)3408 static int validate_call(struct objtool_file *file,
3409 struct instruction *insn,
3410 struct insn_state *state)
3411 {
3412 if (state->noinstr && state->instr <= 0 &&
3413 !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3414 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3415 return 1;
3416 }
3417
3418 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3419 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3420 return 1;
3421 }
3422
3423 if (state->df) {
3424 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3425 return 1;
3426 }
3427
3428 return 0;
3429 }
3430
validate_sibling_call(struct objtool_file * file,struct instruction * insn,struct insn_state * state)3431 static int validate_sibling_call(struct objtool_file *file,
3432 struct instruction *insn,
3433 struct insn_state *state)
3434 {
3435 if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3436 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3437 return 1;
3438 }
3439
3440 return validate_call(file, insn, state);
3441 }
3442
validate_return(struct symbol * func,struct instruction * insn,struct insn_state * state)3443 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3444 {
3445 if (state->noinstr && state->instr > 0) {
3446 WARN_INSN(insn, "return with instrumentation enabled");
3447 return 1;
3448 }
3449
3450 if (state->uaccess && !func_uaccess_safe(func)) {
3451 WARN_INSN(insn, "return with UACCESS enabled");
3452 return 1;
3453 }
3454
3455 if (!state->uaccess && func_uaccess_safe(func)) {
3456 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3457 return 1;
3458 }
3459
3460 if (state->df) {
3461 WARN_INSN(insn, "return with DF set");
3462 return 1;
3463 }
3464
3465 if (func && has_modified_stack_frame(insn, state)) {
3466 WARN_INSN(insn, "return with modified stack frame");
3467 return 1;
3468 }
3469
3470 if (state->cfi.bp_scratch) {
3471 WARN_INSN(insn, "BP used as a scratch register");
3472 return 1;
3473 }
3474
3475 return 0;
3476 }
3477
next_insn_to_validate(struct objtool_file * file,struct instruction * insn)3478 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3479 struct instruction *insn)
3480 {
3481 struct alt_group *alt_group = insn->alt_group;
3482
3483 /*
3484 * Simulate the fact that alternatives are patched in-place. When the
3485 * end of a replacement alt_group is reached, redirect objtool flow to
3486 * the end of the original alt_group.
3487 *
3488 * insn->alts->insn -> alt_group->first_insn
3489 * ...
3490 * alt_group->last_insn
3491 * [alt_group->nop] -> next(orig_group->last_insn)
3492 */
3493 if (alt_group) {
3494 if (alt_group->nop) {
3495 /* ->nop implies ->orig_group */
3496 if (insn == alt_group->last_insn)
3497 return alt_group->nop;
3498 if (insn == alt_group->nop)
3499 goto next_orig;
3500 }
3501 if (insn == alt_group->last_insn && alt_group->orig_group)
3502 goto next_orig;
3503 }
3504
3505 return next_insn_same_sec(file, insn);
3506
3507 next_orig:
3508 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3509 }
3510
skip_alt_group(struct instruction * insn)3511 static bool skip_alt_group(struct instruction *insn)
3512 {
3513 struct instruction *alt_insn = insn->alts ? insn->alts->insn : NULL;
3514
3515 /* ANNOTATE_IGNORE_ALTERNATIVE */
3516 if (insn->alt_group && insn->alt_group->ignore)
3517 return true;
3518
3519 /*
3520 * For NOP patched with CLAC/STAC, only follow the latter to avoid
3521 * impossible code paths combining patched CLAC with unpatched STAC
3522 * or vice versa.
3523 *
3524 * ANNOTATE_IGNORE_ALTERNATIVE could have been used here, but Linus
3525 * requested not to do that to avoid hurting .s file readability
3526 * around CLAC/STAC alternative sites.
3527 */
3528
3529 if (!alt_insn)
3530 return false;
3531
3532 /* Don't override ASM_{CLAC,STAC}_UNSAFE */
3533 if (alt_insn->alt_group && alt_insn->alt_group->ignore)
3534 return false;
3535
3536 return alt_insn->type == INSN_CLAC || alt_insn->type == INSN_STAC;
3537 }
3538
3539 /*
3540 * Follow the branch starting at the given instruction, and recursively follow
3541 * any other branches (jumps). Meanwhile, track the frame pointer state at
3542 * each instruction and validate all the rules described in
3543 * tools/objtool/Documentation/objtool.txt.
3544 */
validate_branch(struct objtool_file * file,struct symbol * func,struct instruction * insn,struct insn_state state)3545 static int validate_branch(struct objtool_file *file, struct symbol *func,
3546 struct instruction *insn, struct insn_state state)
3547 {
3548 struct alternative *alt;
3549 struct instruction *next_insn, *prev_insn = NULL;
3550 struct section *sec;
3551 u8 visited;
3552 int ret;
3553
3554 if (func && func->ignore)
3555 return 0;
3556
3557 sec = insn->sec;
3558
3559 while (1) {
3560 next_insn = next_insn_to_validate(file, insn);
3561
3562 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3563 /* Ignore KCFI type preambles, which always fall through */
3564 if (!strncmp(func->name, "__cfi_", 6) ||
3565 !strncmp(func->name, "__pfx_", 6))
3566 return 0;
3567
3568 if (file->ignore_unreachables)
3569 return 0;
3570
3571 WARN("%s() falls through to next function %s()",
3572 func->name, insn_func(insn)->name);
3573 func->warned = 1;
3574
3575 return 1;
3576 }
3577
3578 visited = VISITED_BRANCH << state.uaccess;
3579 if (insn->visited & VISITED_BRANCH_MASK) {
3580 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3581 return 1;
3582
3583 if (insn->visited & visited)
3584 return 0;
3585 } else {
3586 nr_insns_visited++;
3587 }
3588
3589 if (state.noinstr)
3590 state.instr += insn->instr;
3591
3592 if (insn->hint) {
3593 if (insn->restore) {
3594 struct instruction *save_insn, *i;
3595
3596 i = insn;
3597 save_insn = NULL;
3598
3599 sym_for_each_insn_continue_reverse(file, func, i) {
3600 if (i->save) {
3601 save_insn = i;
3602 break;
3603 }
3604 }
3605
3606 if (!save_insn) {
3607 WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3608 return 1;
3609 }
3610
3611 if (!save_insn->visited) {
3612 /*
3613 * If the restore hint insn is at the
3614 * beginning of a basic block and was
3615 * branched to from elsewhere, and the
3616 * save insn hasn't been visited yet,
3617 * defer following this branch for now.
3618 * It will be seen later via the
3619 * straight-line path.
3620 */
3621 if (!prev_insn)
3622 return 0;
3623
3624 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3625 return 1;
3626 }
3627
3628 insn->cfi = save_insn->cfi;
3629 nr_cfi_reused++;
3630 }
3631
3632 state.cfi = *insn->cfi;
3633 } else {
3634 /* XXX track if we actually changed state.cfi */
3635
3636 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3637 insn->cfi = prev_insn->cfi;
3638 nr_cfi_reused++;
3639 } else {
3640 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3641 }
3642 }
3643
3644 insn->visited |= visited;
3645
3646 if (propagate_alt_cfi(file, insn))
3647 return 1;
3648
3649 if (insn->alts) {
3650 for (alt = insn->alts; alt; alt = alt->next) {
3651 ret = validate_branch(file, func, alt->insn, state);
3652 if (ret) {
3653 BT_INSN(insn, "(alt)");
3654 return ret;
3655 }
3656 }
3657 }
3658
3659 if (skip_alt_group(insn))
3660 return 0;
3661
3662 if (handle_insn_ops(insn, next_insn, &state))
3663 return 1;
3664
3665 switch (insn->type) {
3666
3667 case INSN_RETURN:
3668 return validate_return(func, insn, &state);
3669
3670 case INSN_CALL:
3671 case INSN_CALL_DYNAMIC:
3672 ret = validate_call(file, insn, &state);
3673 if (ret)
3674 return ret;
3675
3676 if (opts.stackval && func && !is_special_call(insn) &&
3677 !has_valid_stack_frame(&state)) {
3678 WARN_INSN(insn, "call without frame pointer save/setup");
3679 return 1;
3680 }
3681
3682 break;
3683
3684 case INSN_JUMP_CONDITIONAL:
3685 case INSN_JUMP_UNCONDITIONAL:
3686 if (is_sibling_call(insn)) {
3687 ret = validate_sibling_call(file, insn, &state);
3688 if (ret)
3689 return ret;
3690
3691 } else if (insn->jump_dest) {
3692 ret = validate_branch(file, func,
3693 insn->jump_dest, state);
3694 if (ret) {
3695 BT_INSN(insn, "(branch)");
3696 return ret;
3697 }
3698 }
3699
3700 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3701 return 0;
3702
3703 break;
3704
3705 case INSN_JUMP_DYNAMIC:
3706 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3707 if (is_sibling_call(insn)) {
3708 ret = validate_sibling_call(file, insn, &state);
3709 if (ret)
3710 return ret;
3711 }
3712
3713 if (insn->type == INSN_JUMP_DYNAMIC)
3714 return 0;
3715
3716 break;
3717
3718 case INSN_SYSCALL:
3719 if (func && (!next_insn || !next_insn->hint)) {
3720 WARN_INSN(insn, "unsupported instruction in callable function");
3721 return 1;
3722 }
3723
3724 break;
3725
3726 case INSN_SYSRET:
3727 if (func && (!next_insn || !next_insn->hint)) {
3728 WARN_INSN(insn, "unsupported instruction in callable function");
3729 return 1;
3730 }
3731
3732 return 0;
3733
3734 case INSN_STAC:
3735 if (!opts.uaccess)
3736 break;
3737
3738 if (state.uaccess) {
3739 WARN_INSN(insn, "recursive UACCESS enable");
3740 return 1;
3741 }
3742
3743 state.uaccess = true;
3744 break;
3745
3746 case INSN_CLAC:
3747 if (!opts.uaccess)
3748 break;
3749
3750 if (!state.uaccess && func) {
3751 WARN_INSN(insn, "redundant UACCESS disable");
3752 return 1;
3753 }
3754
3755 if (func_uaccess_safe(func) && !state.uaccess_stack) {
3756 WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3757 return 1;
3758 }
3759
3760 state.uaccess = false;
3761 break;
3762
3763 case INSN_STD:
3764 if (state.df) {
3765 WARN_INSN(insn, "recursive STD");
3766 return 1;
3767 }
3768
3769 state.df = true;
3770 break;
3771
3772 case INSN_CLD:
3773 if (!state.df && func) {
3774 WARN_INSN(insn, "redundant CLD");
3775 return 1;
3776 }
3777
3778 state.df = false;
3779 break;
3780
3781 default:
3782 break;
3783 }
3784
3785 if (insn->dead_end)
3786 return 0;
3787
3788 if (!next_insn) {
3789 if (state.cfi.cfa.base == CFI_UNDEFINED)
3790 return 0;
3791 if (file->ignore_unreachables)
3792 return 0;
3793
3794 WARN("%s%sunexpected end of section %s",
3795 func ? func->name : "", func ? "(): " : "",
3796 sec->name);
3797 return 1;
3798 }
3799
3800 prev_insn = insn;
3801 insn = next_insn;
3802 }
3803
3804 return 0;
3805 }
3806
validate_unwind_hint(struct objtool_file * file,struct instruction * insn,struct insn_state * state)3807 static int validate_unwind_hint(struct objtool_file *file,
3808 struct instruction *insn,
3809 struct insn_state *state)
3810 {
3811 if (insn->hint && !insn->visited) {
3812 int ret = validate_branch(file, insn_func(insn), insn, *state);
3813 if (ret)
3814 BT_INSN(insn, "<=== (hint)");
3815 return ret;
3816 }
3817
3818 return 0;
3819 }
3820
validate_unwind_hints(struct objtool_file * file,struct section * sec)3821 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3822 {
3823 struct instruction *insn;
3824 struct insn_state state;
3825 int warnings = 0;
3826
3827 if (!file->hints)
3828 return 0;
3829
3830 init_insn_state(file, &state, sec);
3831
3832 if (sec) {
3833 sec_for_each_insn(file, sec, insn)
3834 warnings += validate_unwind_hint(file, insn, &state);
3835 } else {
3836 for_each_insn(file, insn)
3837 warnings += validate_unwind_hint(file, insn, &state);
3838 }
3839
3840 return warnings;
3841 }
3842
3843 /*
3844 * Validate rethunk entry constraint: must untrain RET before the first RET.
3845 *
3846 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3847 * before an actual RET instruction.
3848 */
validate_unret(struct objtool_file * file,struct instruction * insn)3849 static int validate_unret(struct objtool_file *file, struct instruction *insn)
3850 {
3851 struct instruction *next, *dest;
3852 int ret;
3853
3854 for (;;) {
3855 next = next_insn_to_validate(file, insn);
3856
3857 if (insn->visited & VISITED_UNRET)
3858 return 0;
3859
3860 insn->visited |= VISITED_UNRET;
3861
3862 if (insn->alts) {
3863 struct alternative *alt;
3864 for (alt = insn->alts; alt; alt = alt->next) {
3865 ret = validate_unret(file, alt->insn);
3866 if (ret) {
3867 BT_INSN(insn, "(alt)");
3868 return ret;
3869 }
3870 }
3871 }
3872
3873 switch (insn->type) {
3874
3875 case INSN_CALL_DYNAMIC:
3876 case INSN_JUMP_DYNAMIC:
3877 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3878 WARN_INSN(insn, "early indirect call");
3879 return 1;
3880
3881 case INSN_JUMP_UNCONDITIONAL:
3882 case INSN_JUMP_CONDITIONAL:
3883 if (!is_sibling_call(insn)) {
3884 if (!insn->jump_dest) {
3885 WARN_INSN(insn, "unresolved jump target after linking?!?");
3886 return 1;
3887 }
3888 ret = validate_unret(file, insn->jump_dest);
3889 if (ret) {
3890 BT_INSN(insn, "(branch%s)",
3891 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3892 return ret;
3893 }
3894
3895 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3896 return 0;
3897
3898 break;
3899 }
3900
3901 /* fallthrough */
3902 case INSN_CALL:
3903 dest = find_insn(file, insn_call_dest(insn)->sec,
3904 insn_call_dest(insn)->offset);
3905 if (!dest) {
3906 WARN("Unresolved function after linking!?: %s",
3907 insn_call_dest(insn)->name);
3908 return 1;
3909 }
3910
3911 ret = validate_unret(file, dest);
3912 if (ret) {
3913 BT_INSN(insn, "(call)");
3914 return ret;
3915 }
3916 /*
3917 * If a call returns without error, it must have seen UNTRAIN_RET.
3918 * Therefore any non-error return is a success.
3919 */
3920 return 0;
3921
3922 case INSN_RETURN:
3923 WARN_INSN(insn, "RET before UNTRAIN");
3924 return 1;
3925
3926 case INSN_SYSCALL:
3927 break;
3928
3929 case INSN_SYSRET:
3930 return 0;
3931
3932 case INSN_NOP:
3933 if (insn->retpoline_safe)
3934 return 0;
3935 break;
3936
3937 default:
3938 break;
3939 }
3940
3941 if (insn->dead_end)
3942 return 0;
3943
3944 if (!next) {
3945 WARN_INSN(insn, "teh end!");
3946 return 1;
3947 }
3948 insn = next;
3949 }
3950
3951 return 0;
3952 }
3953
3954 /*
3955 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3956 * VALIDATE_UNRET_END before RET.
3957 */
validate_unrets(struct objtool_file * file)3958 static int validate_unrets(struct objtool_file *file)
3959 {
3960 struct instruction *insn;
3961 int warnings = 0;
3962
3963 for_each_insn(file, insn) {
3964 if (!insn->unret)
3965 continue;
3966
3967 warnings += validate_unret(file, insn);
3968 }
3969
3970 return warnings;
3971 }
3972
validate_retpoline(struct objtool_file * file)3973 static int validate_retpoline(struct objtool_file *file)
3974 {
3975 struct instruction *insn;
3976 int warnings = 0;
3977
3978 for_each_insn(file, insn) {
3979 if (insn->type != INSN_JUMP_DYNAMIC &&
3980 insn->type != INSN_CALL_DYNAMIC &&
3981 insn->type != INSN_RETURN)
3982 continue;
3983
3984 if (insn->retpoline_safe)
3985 continue;
3986
3987 if (insn->sec->init)
3988 continue;
3989
3990 if (insn->type == INSN_RETURN) {
3991 if (opts.rethunk) {
3992 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
3993 warnings++;
3994 }
3995 continue;
3996 }
3997
3998 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
3999 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4000 warnings++;
4001 }
4002
4003 return warnings;
4004 }
4005
is_kasan_insn(struct instruction * insn)4006 static bool is_kasan_insn(struct instruction *insn)
4007 {
4008 return (insn->type == INSN_CALL &&
4009 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4010 }
4011
is_ubsan_insn(struct instruction * insn)4012 static bool is_ubsan_insn(struct instruction *insn)
4013 {
4014 return (insn->type == INSN_CALL &&
4015 !strcmp(insn_call_dest(insn)->name,
4016 "__ubsan_handle_builtin_unreachable"));
4017 }
4018
ignore_unreachable_insn(struct objtool_file * file,struct instruction * insn)4019 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4020 {
4021 struct symbol *func = insn_func(insn);
4022 struct instruction *prev_insn;
4023 int i;
4024
4025 if (insn->type == INSN_NOP || insn->type == INSN_TRAP || (func && func->ignore))
4026 return true;
4027
4028 /*
4029 * Ignore alternative replacement instructions. This can happen
4030 * when a whitelisted function uses one of the ALTERNATIVE macros.
4031 */
4032 if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4033 !strcmp(insn->sec->name, ".altinstr_aux"))
4034 return true;
4035
4036 /*
4037 * Whole archive runs might encounter dead code from weak symbols.
4038 * This is where the linker will have dropped the weak symbol in
4039 * favour of a regular symbol, but leaves the code in place.
4040 *
4041 * In this case we'll find a piece of code (whole function) that is not
4042 * covered by a !section symbol. Ignore them.
4043 */
4044 if (opts.link && !func) {
4045 int size = find_symbol_hole_containing(insn->sec, insn->offset);
4046 unsigned long end = insn->offset + size;
4047
4048 if (!size) /* not a hole */
4049 return false;
4050
4051 if (size < 0) /* hole until the end */
4052 return true;
4053
4054 sec_for_each_insn_continue(file, insn) {
4055 /*
4056 * If we reach a visited instruction at or before the
4057 * end of the hole, ignore the unreachable.
4058 */
4059 if (insn->visited)
4060 return true;
4061
4062 if (insn->offset >= end)
4063 break;
4064
4065 /*
4066 * If this hole jumps to a .cold function, mark it ignore too.
4067 */
4068 if (insn->jump_dest && insn_func(insn->jump_dest) &&
4069 strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4070 insn_func(insn->jump_dest)->ignore = true;
4071 }
4072 }
4073
4074 return false;
4075 }
4076
4077 if (!func)
4078 return false;
4079
4080 if (func->static_call_tramp)
4081 return true;
4082
4083 /*
4084 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4085 * __builtin_unreachable(). The BUG() macro has an unreachable() after
4086 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4087 * (or occasionally a JMP to UD2).
4088 *
4089 * It may also insert a UD2 after calling a __noreturn function.
4090 */
4091 prev_insn = prev_insn_same_sec(file, insn);
4092 if (prev_insn && prev_insn->dead_end &&
4093 (insn->type == INSN_BUG ||
4094 (insn->type == INSN_JUMP_UNCONDITIONAL &&
4095 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4096 return true;
4097
4098 /*
4099 * Check if this (or a subsequent) instruction is related to
4100 * CONFIG_UBSAN or CONFIG_KASAN.
4101 *
4102 * End the search at 5 instructions to avoid going into the weeds.
4103 */
4104 for (i = 0; i < 5; i++) {
4105
4106 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4107 return true;
4108
4109 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4110 if (insn->jump_dest &&
4111 insn_func(insn->jump_dest) == func) {
4112 insn = insn->jump_dest;
4113 continue;
4114 }
4115
4116 break;
4117 }
4118
4119 if (insn->offset + insn->len >= func->offset + func->len)
4120 break;
4121
4122 insn = next_insn_same_sec(file, insn);
4123 }
4124
4125 return false;
4126 }
4127
add_prefix_symbol(struct objtool_file * file,struct symbol * func)4128 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4129 {
4130 struct instruction *insn, *prev;
4131 struct cfi_state *cfi;
4132
4133 insn = find_insn(file, func->sec, func->offset);
4134 if (!insn)
4135 return -1;
4136
4137 for (prev = prev_insn_same_sec(file, insn);
4138 prev;
4139 prev = prev_insn_same_sec(file, prev)) {
4140 u64 offset;
4141
4142 if (prev->type != INSN_NOP)
4143 return -1;
4144
4145 offset = func->offset - prev->offset;
4146
4147 if (offset > opts.prefix)
4148 return -1;
4149
4150 if (offset < opts.prefix)
4151 continue;
4152
4153 elf_create_prefix_symbol(file->elf, func, opts.prefix);
4154 break;
4155 }
4156
4157 if (!prev)
4158 return -1;
4159
4160 if (!insn->cfi) {
4161 /*
4162 * This can happen if stack validation isn't enabled or the
4163 * function is annotated with STACK_FRAME_NON_STANDARD.
4164 */
4165 return 0;
4166 }
4167
4168 /* Propagate insn->cfi to the prefix code */
4169 cfi = cfi_hash_find_or_add(insn->cfi);
4170 for (; prev != insn; prev = next_insn_same_sec(file, prev))
4171 prev->cfi = cfi;
4172
4173 return 0;
4174 }
4175
add_prefix_symbols(struct objtool_file * file)4176 static int add_prefix_symbols(struct objtool_file *file)
4177 {
4178 struct section *sec;
4179 struct symbol *func;
4180
4181 for_each_sec(file, sec) {
4182 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4183 continue;
4184
4185 sec_for_each_sym(sec, func) {
4186 if (func->type != STT_FUNC)
4187 continue;
4188
4189 add_prefix_symbol(file, func);
4190 }
4191 }
4192
4193 return 0;
4194 }
4195
validate_symbol(struct objtool_file * file,struct section * sec,struct symbol * sym,struct insn_state * state)4196 static int validate_symbol(struct objtool_file *file, struct section *sec,
4197 struct symbol *sym, struct insn_state *state)
4198 {
4199 struct instruction *insn;
4200 int ret;
4201
4202 if (!sym->len) {
4203 WARN("%s() is missing an ELF size annotation", sym->name);
4204 return 1;
4205 }
4206
4207 if (sym->pfunc != sym || sym->alias != sym)
4208 return 0;
4209
4210 insn = find_insn(file, sec, sym->offset);
4211 if (!insn || insn->visited)
4212 return 0;
4213
4214 if (opts.uaccess)
4215 state->uaccess = sym->uaccess_safe;
4216
4217 ret = validate_branch(file, insn_func(insn), insn, *state);
4218 if (ret)
4219 BT_INSN(insn, "<=== (sym)");
4220 return ret;
4221 }
4222
validate_section(struct objtool_file * file,struct section * sec)4223 static int validate_section(struct objtool_file *file, struct section *sec)
4224 {
4225 struct insn_state state;
4226 struct symbol *func;
4227 int warnings = 0;
4228
4229 sec_for_each_sym(sec, func) {
4230 if (func->type != STT_FUNC)
4231 continue;
4232
4233 init_insn_state(file, &state, sec);
4234 set_func_state(&state.cfi);
4235
4236 warnings += validate_symbol(file, sec, func, &state);
4237 }
4238
4239 return warnings;
4240 }
4241
validate_noinstr_sections(struct objtool_file * file)4242 static int validate_noinstr_sections(struct objtool_file *file)
4243 {
4244 struct section *sec;
4245 int warnings = 0;
4246
4247 sec = find_section_by_name(file->elf, ".noinstr.text");
4248 if (sec) {
4249 warnings += validate_section(file, sec);
4250 warnings += validate_unwind_hints(file, sec);
4251 }
4252
4253 sec = find_section_by_name(file->elf, ".entry.text");
4254 if (sec) {
4255 warnings += validate_section(file, sec);
4256 warnings += validate_unwind_hints(file, sec);
4257 }
4258
4259 sec = find_section_by_name(file->elf, ".cpuidle.text");
4260 if (sec) {
4261 warnings += validate_section(file, sec);
4262 warnings += validate_unwind_hints(file, sec);
4263 }
4264
4265 return warnings;
4266 }
4267
validate_functions(struct objtool_file * file)4268 static int validate_functions(struct objtool_file *file)
4269 {
4270 struct section *sec;
4271 int warnings = 0;
4272
4273 for_each_sec(file, sec) {
4274 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4275 continue;
4276
4277 warnings += validate_section(file, sec);
4278 }
4279
4280 return warnings;
4281 }
4282
mark_endbr_used(struct instruction * insn)4283 static void mark_endbr_used(struct instruction *insn)
4284 {
4285 if (!list_empty(&insn->call_node))
4286 list_del_init(&insn->call_node);
4287 }
4288
noendbr_range(struct objtool_file * file,struct instruction * insn)4289 static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4290 {
4291 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4292 struct instruction *first;
4293
4294 if (!sym)
4295 return false;
4296
4297 first = find_insn(file, sym->sec, sym->offset);
4298 if (!first)
4299 return false;
4300
4301 if (first->type != INSN_ENDBR && !first->noendbr)
4302 return false;
4303
4304 return insn->offset == sym->offset + sym->len;
4305 }
4306
__validate_ibt_insn(struct objtool_file * file,struct instruction * insn,struct instruction * dest)4307 static int __validate_ibt_insn(struct objtool_file *file, struct instruction *insn,
4308 struct instruction *dest)
4309 {
4310 if (dest->type == INSN_ENDBR) {
4311 mark_endbr_used(dest);
4312 return 0;
4313 }
4314
4315 if (insn_func(dest) && insn_func(insn) &&
4316 insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4317 /*
4318 * Anything from->to self is either _THIS_IP_ or
4319 * IRET-to-self.
4320 *
4321 * There is no sane way to annotate _THIS_IP_ since the
4322 * compiler treats the relocation as a constant and is
4323 * happy to fold in offsets, skewing any annotation we
4324 * do, leading to vast amounts of false-positives.
4325 *
4326 * There's also compiler generated _THIS_IP_ through
4327 * KCOV and such which we have no hope of annotating.
4328 *
4329 * As such, blanket accept self-references without
4330 * issue.
4331 */
4332 return 0;
4333 }
4334
4335 /*
4336 * Accept anything ANNOTATE_NOENDBR.
4337 */
4338 if (dest->noendbr)
4339 return 0;
4340
4341 /*
4342 * Accept if this is the instruction after a symbol
4343 * that is (no)endbr -- typical code-range usage.
4344 */
4345 if (noendbr_range(file, dest))
4346 return 0;
4347
4348 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4349 return 1;
4350 }
4351
validate_ibt_insn(struct objtool_file * file,struct instruction * insn)4352 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4353 {
4354 struct instruction *dest;
4355 struct reloc *reloc;
4356 unsigned long off;
4357 int warnings = 0;
4358
4359 /*
4360 * Looking for function pointer load relocations. Ignore
4361 * direct/indirect branches:
4362 */
4363 switch (insn->type) {
4364
4365 case INSN_CALL:
4366 case INSN_CALL_DYNAMIC:
4367 case INSN_JUMP_CONDITIONAL:
4368 case INSN_JUMP_UNCONDITIONAL:
4369 case INSN_JUMP_DYNAMIC:
4370 case INSN_JUMP_DYNAMIC_CONDITIONAL:
4371 case INSN_RETURN:
4372 case INSN_NOP:
4373 return 0;
4374
4375 case INSN_LEA_RIP:
4376 if (!insn_reloc(file, insn)) {
4377 /* local function pointer reference without reloc */
4378
4379 off = arch_jump_destination(insn);
4380
4381 dest = find_insn(file, insn->sec, off);
4382 if (!dest) {
4383 WARN_INSN(insn, "corrupt function pointer reference");
4384 return 1;
4385 }
4386
4387 return __validate_ibt_insn(file, insn, dest);
4388 }
4389 break;
4390
4391 default:
4392 break;
4393 }
4394
4395 for (reloc = insn_reloc(file, insn);
4396 reloc;
4397 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4398 reloc_offset(reloc) + 1,
4399 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4400
4401 off = reloc->sym->offset;
4402 if (reloc_type(reloc) == R_X86_64_PC32 ||
4403 reloc_type(reloc) == R_X86_64_PLT32)
4404 off += arch_dest_reloc_offset(reloc_addend(reloc));
4405 else
4406 off += reloc_addend(reloc);
4407
4408 dest = find_insn(file, reloc->sym->sec, off);
4409 if (!dest)
4410 continue;
4411
4412 warnings += __validate_ibt_insn(file, insn, dest);
4413 }
4414
4415 return warnings;
4416 }
4417
validate_ibt_data_reloc(struct objtool_file * file,struct reloc * reloc)4418 static int validate_ibt_data_reloc(struct objtool_file *file,
4419 struct reloc *reloc)
4420 {
4421 struct instruction *dest;
4422
4423 dest = find_insn(file, reloc->sym->sec,
4424 reloc->sym->offset + reloc_addend(reloc));
4425 if (!dest)
4426 return 0;
4427
4428 if (dest->type == INSN_ENDBR) {
4429 mark_endbr_used(dest);
4430 return 0;
4431 }
4432
4433 if (dest->noendbr)
4434 return 0;
4435
4436 WARN_FUNC(reloc->sec->base, reloc_offset(reloc),
4437 "data relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4438
4439 return 1;
4440 }
4441
4442 /*
4443 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4444 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4445 * NOPs) later, in create_ibt_endbr_seal_sections().
4446 */
validate_ibt(struct objtool_file * file)4447 static int validate_ibt(struct objtool_file *file)
4448 {
4449 struct section *sec;
4450 struct reloc *reloc;
4451 struct instruction *insn;
4452 int warnings = 0;
4453
4454 for_each_insn(file, insn)
4455 warnings += validate_ibt_insn(file, insn);
4456
4457 for_each_sec(file, sec) {
4458
4459 /* Already done by validate_ibt_insn() */
4460 if (sec->sh.sh_flags & SHF_EXECINSTR)
4461 continue;
4462
4463 if (!sec->rsec)
4464 continue;
4465
4466 /*
4467 * These sections can reference text addresses, but not with
4468 * the intent to indirect branch to them.
4469 */
4470 if ((!strncmp(sec->name, ".discard", 8) &&
4471 strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
4472 !strncmp(sec->name, ".debug", 6) ||
4473 !strcmp(sec->name, ".altinstructions") ||
4474 !strcmp(sec->name, ".ibt_endbr_seal") ||
4475 !strcmp(sec->name, ".orc_unwind_ip") ||
4476 !strcmp(sec->name, ".parainstructions") ||
4477 !strcmp(sec->name, ".retpoline_sites") ||
4478 !strcmp(sec->name, ".smp_locks") ||
4479 !strcmp(sec->name, ".static_call_sites") ||
4480 !strcmp(sec->name, "_error_injection_whitelist") ||
4481 !strcmp(sec->name, "_kprobe_blacklist") ||
4482 !strcmp(sec->name, "__bug_table") ||
4483 !strcmp(sec->name, "__ex_table") ||
4484 !strcmp(sec->name, "__jump_table") ||
4485 !strcmp(sec->name, "__mcount_loc") ||
4486 !strcmp(sec->name, ".kcfi_traps") ||
4487 !strcmp(sec->name, ".llvm.call-graph-profile") ||
4488 !strcmp(sec->name, ".llvm_bb_addr_map") ||
4489 !strcmp(sec->name, "__tracepoints") ||
4490 strstr(sec->name, "__patchable_function_entries"))
4491 continue;
4492
4493 for_each_reloc(sec->rsec, reloc)
4494 warnings += validate_ibt_data_reloc(file, reloc);
4495 }
4496
4497 return warnings;
4498 }
4499
validate_sls(struct objtool_file * file)4500 static int validate_sls(struct objtool_file *file)
4501 {
4502 struct instruction *insn, *next_insn;
4503 int warnings = 0;
4504
4505 for_each_insn(file, insn) {
4506 next_insn = next_insn_same_sec(file, insn);
4507
4508 if (insn->retpoline_safe)
4509 continue;
4510
4511 switch (insn->type) {
4512 case INSN_RETURN:
4513 if (!next_insn || next_insn->type != INSN_TRAP) {
4514 WARN_INSN(insn, "missing int3 after ret");
4515 warnings++;
4516 }
4517
4518 break;
4519 case INSN_JUMP_DYNAMIC:
4520 if (!next_insn || next_insn->type != INSN_TRAP) {
4521 WARN_INSN(insn, "missing int3 after indirect jump");
4522 warnings++;
4523 }
4524 break;
4525 default:
4526 break;
4527 }
4528 }
4529
4530 return warnings;
4531 }
4532
validate_reachable_instructions(struct objtool_file * file)4533 static int validate_reachable_instructions(struct objtool_file *file)
4534 {
4535 struct instruction *insn, *prev_insn;
4536 struct symbol *call_dest;
4537 int warnings = 0;
4538
4539 if (file->ignore_unreachables)
4540 return 0;
4541
4542 for_each_insn(file, insn) {
4543 if (insn->visited || ignore_unreachable_insn(file, insn))
4544 continue;
4545
4546 prev_insn = prev_insn_same_sec(file, insn);
4547 if (prev_insn && prev_insn->dead_end) {
4548 call_dest = insn_call_dest(prev_insn);
4549 if (call_dest) {
4550 WARN_INSN(insn, "%s() missing __noreturn in .c/.h or NORETURN() in noreturns.h",
4551 call_dest->name);
4552 warnings++;
4553 continue;
4554 }
4555 }
4556
4557 WARN_INSN(insn, "unreachable instruction");
4558 warnings++;
4559 }
4560
4561 return warnings;
4562 }
4563
4564 /* 'funcs' is a space-separated list of function names */
disas_funcs(const char * funcs)4565 static void disas_funcs(const char *funcs)
4566 {
4567 const char *objdump_str, *cross_compile;
4568 int size, ret;
4569 char *cmd;
4570
4571 cross_compile = getenv("CROSS_COMPILE");
4572 if (!cross_compile)
4573 cross_compile = "";
4574
4575 objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4576 "BEGIN { split(_funcs, funcs); }"
4577 "/^$/ { func_match = 0; }"
4578 "/<.*>:/ { "
4579 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4580 "for (i in funcs) {"
4581 "if (funcs[i] == f) {"
4582 "func_match = 1;"
4583 "base = strtonum(\"0x\" $1);"
4584 "break;"
4585 "}"
4586 "}"
4587 "}"
4588 "{"
4589 "if (func_match) {"
4590 "addr = strtonum(\"0x\" $1);"
4591 "printf(\"%%04x \", addr - base);"
4592 "print;"
4593 "}"
4594 "}' 1>&2";
4595
4596 /* fake snprintf() to calculate the size */
4597 size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4598 if (size <= 0) {
4599 WARN("objdump string size calculation failed");
4600 return;
4601 }
4602
4603 cmd = malloc(size);
4604
4605 /* real snprintf() */
4606 snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4607 ret = system(cmd);
4608 if (ret) {
4609 WARN("disassembly failed: %d", ret);
4610 return;
4611 }
4612 }
4613
disas_warned_funcs(struct objtool_file * file)4614 static void disas_warned_funcs(struct objtool_file *file)
4615 {
4616 struct symbol *sym;
4617 char *funcs = NULL, *tmp;
4618
4619 for_each_sym(file, sym) {
4620 if (sym->warned) {
4621 if (!funcs) {
4622 funcs = malloc(strlen(sym->name) + 1);
4623 if (!funcs) {
4624 ERROR_GLIBC("malloc");
4625 return;
4626 }
4627 strcpy(funcs, sym->name);
4628 } else {
4629 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4630 if (!tmp) {
4631 ERROR_GLIBC("malloc");
4632 return;
4633 }
4634 sprintf(tmp, "%s %s", funcs, sym->name);
4635 free(funcs);
4636 funcs = tmp;
4637 }
4638 }
4639 }
4640
4641 if (funcs)
4642 disas_funcs(funcs);
4643 }
4644
4645 struct insn_chunk {
4646 void *addr;
4647 struct insn_chunk *next;
4648 };
4649
4650 /*
4651 * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4652 * which can trigger more allocations for .debug_* sections whose data hasn't
4653 * been read yet.
4654 */
free_insns(struct objtool_file * file)4655 static void free_insns(struct objtool_file *file)
4656 {
4657 struct instruction *insn;
4658 struct insn_chunk *chunks = NULL, *chunk;
4659
4660 for_each_insn(file, insn) {
4661 if (!insn->idx) {
4662 chunk = malloc(sizeof(*chunk));
4663 chunk->addr = insn;
4664 chunk->next = chunks;
4665 chunks = chunk;
4666 }
4667 }
4668
4669 for (chunk = chunks; chunk; chunk = chunk->next)
4670 free(chunk->addr);
4671 }
4672
check(struct objtool_file * file)4673 int check(struct objtool_file *file)
4674 {
4675 int ret = 0, warnings = 0;
4676
4677 arch_initial_func_cfi_state(&initial_func_cfi);
4678 init_cfi_state(&init_cfi);
4679 init_cfi_state(&func_cfi);
4680 set_func_state(&func_cfi);
4681 init_cfi_state(&force_undefined_cfi);
4682 force_undefined_cfi.force_undefined = true;
4683
4684 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) {
4685 ret = -1;
4686 goto out;
4687 }
4688
4689 cfi_hash_add(&init_cfi);
4690 cfi_hash_add(&func_cfi);
4691
4692 ret = decode_sections(file);
4693 if (ret)
4694 goto out;
4695
4696 if (!nr_insns)
4697 goto out;
4698
4699 if (opts.retpoline)
4700 warnings += validate_retpoline(file);
4701
4702 if (opts.stackval || opts.orc || opts.uaccess) {
4703 int w = 0;
4704
4705 w += validate_functions(file);
4706 w += validate_unwind_hints(file, NULL);
4707 if (!w)
4708 w += validate_reachable_instructions(file);
4709
4710 warnings += w;
4711
4712 } else if (opts.noinstr) {
4713 warnings += validate_noinstr_sections(file);
4714 }
4715
4716 if (opts.unret) {
4717 /*
4718 * Must be after validate_branch() and friends, it plays
4719 * further games with insn->visited.
4720 */
4721 warnings += validate_unrets(file);
4722 }
4723
4724 if (opts.ibt)
4725 warnings += validate_ibt(file);
4726
4727 if (opts.sls)
4728 warnings += validate_sls(file);
4729
4730 if (opts.static_call) {
4731 ret = create_static_call_sections(file);
4732 if (ret)
4733 goto out;
4734 }
4735
4736 if (opts.retpoline) {
4737 ret = create_retpoline_sites_sections(file);
4738 if (ret)
4739 goto out;
4740 }
4741
4742 if (opts.cfi) {
4743 ret = create_cfi_sections(file);
4744 if (ret)
4745 goto out;
4746 }
4747
4748 if (opts.rethunk) {
4749 ret = create_return_sites_sections(file);
4750 if (ret)
4751 goto out;
4752
4753 if (opts.hack_skylake) {
4754 ret = create_direct_call_sections(file);
4755 if (ret)
4756 goto out;
4757 }
4758 }
4759
4760 if (opts.mcount) {
4761 ret = create_mcount_loc_sections(file);
4762 if (ret)
4763 goto out;
4764 }
4765
4766 if (opts.prefix) {
4767 ret = add_prefix_symbols(file);
4768 if (ret)
4769 goto out;
4770 }
4771
4772 if (opts.ibt) {
4773 ret = create_ibt_endbr_seal_sections(file);
4774 if (ret)
4775 goto out;
4776 }
4777
4778 if (opts.orc && nr_insns) {
4779 ret = orc_create(file);
4780 if (ret)
4781 goto out;
4782 }
4783
4784 free_insns(file);
4785
4786 if (opts.stats) {
4787 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4788 printf("nr_cfi: %ld\n", nr_cfi);
4789 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4790 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4791 }
4792
4793 out:
4794 if (!ret && !warnings)
4795 return 0;
4796
4797 if (opts.werror && warnings)
4798 ret = 1;
4799
4800 if (opts.verbose) {
4801 if (opts.werror && warnings)
4802 WARN("%d warning(s) upgraded to errors", warnings);
4803 print_args();
4804 disas_warned_funcs(file);
4805 }
4806
4807 return ret;
4808 }
4809