xref: /linux/tools/objtool/check.c (revision 36f353a1ebf88280f58d1ebfe2731251d9159456)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4  */
5 
6 #include <string.h>
7 #include <stdlib.h>
8 #include <inttypes.h>
9 #include <sys/mman.h>
10 
11 #include <objtool/builtin.h>
12 #include <objtool/cfi.h>
13 #include <objtool/arch.h>
14 #include <objtool/check.h>
15 #include <objtool/special.h>
16 #include <objtool/warn.h>
17 #include <objtool/endianness.h>
18 
19 #include <linux/objtool_types.h>
20 #include <linux/hashtable.h>
21 #include <linux/kernel.h>
22 #include <linux/static_call_types.h>
23 
24 struct alternative {
25 	struct alternative *next;
26 	struct instruction *insn;
27 	bool skip_orig;
28 };
29 
30 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
31 
32 static struct cfi_init_state initial_func_cfi;
33 static struct cfi_state init_cfi;
34 static struct cfi_state func_cfi;
35 static struct cfi_state force_undefined_cfi;
36 
37 struct instruction *find_insn(struct objtool_file *file,
38 			      struct section *sec, unsigned long offset)
39 {
40 	struct instruction *insn;
41 
42 	hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
43 		if (insn->sec == sec && insn->offset == offset)
44 			return insn;
45 	}
46 
47 	return NULL;
48 }
49 
50 struct instruction *next_insn_same_sec(struct objtool_file *file,
51 				       struct instruction *insn)
52 {
53 	if (insn->idx == INSN_CHUNK_MAX)
54 		return find_insn(file, insn->sec, insn->offset + insn->len);
55 
56 	insn++;
57 	if (!insn->len)
58 		return NULL;
59 
60 	return insn;
61 }
62 
63 static struct instruction *next_insn_same_func(struct objtool_file *file,
64 					       struct instruction *insn)
65 {
66 	struct instruction *next = next_insn_same_sec(file, insn);
67 	struct symbol *func = insn_func(insn);
68 
69 	if (!func)
70 		return NULL;
71 
72 	if (next && insn_func(next) == func)
73 		return next;
74 
75 	/* Check if we're already in the subfunction: */
76 	if (func == func->cfunc)
77 		return NULL;
78 
79 	/* Move to the subfunction: */
80 	return find_insn(file, func->cfunc->sec, func->cfunc->offset);
81 }
82 
83 static struct instruction *prev_insn_same_sec(struct objtool_file *file,
84 					      struct instruction *insn)
85 {
86 	if (insn->idx == 0) {
87 		if (insn->prev_len)
88 			return find_insn(file, insn->sec, insn->offset - insn->prev_len);
89 		return NULL;
90 	}
91 
92 	return insn - 1;
93 }
94 
95 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
96 					      struct instruction *insn)
97 {
98 	struct instruction *prev = prev_insn_same_sec(file, insn);
99 
100 	if (prev && insn_func(prev) == insn_func(insn))
101 		return prev;
102 
103 	return NULL;
104 }
105 
106 #define for_each_insn(file, insn)					\
107 	for (struct section *__sec, *__fake = (struct section *)1;	\
108 	     __fake; __fake = NULL)					\
109 		for_each_sec(file, __sec)				\
110 			sec_for_each_insn(file, __sec, insn)
111 
112 #define func_for_each_insn(file, func, insn)				\
113 	for (insn = find_insn(file, func->sec, func->offset);		\
114 	     insn;							\
115 	     insn = next_insn_same_func(file, insn))
116 
117 #define sym_for_each_insn(file, sym, insn)				\
118 	for (insn = find_insn(file, sym->sec, sym->offset);		\
119 	     insn && insn->offset < sym->offset + sym->len;		\
120 	     insn = next_insn_same_sec(file, insn))
121 
122 #define sym_for_each_insn_continue_reverse(file, sym, insn)		\
123 	for (insn = prev_insn_same_sec(file, insn);			\
124 	     insn && insn->offset >= sym->offset;			\
125 	     insn = prev_insn_same_sec(file, insn))
126 
127 #define sec_for_each_insn_from(file, insn)				\
128 	for (; insn; insn = next_insn_same_sec(file, insn))
129 
130 #define sec_for_each_insn_continue(file, insn)				\
131 	for (insn = next_insn_same_sec(file, insn); insn;		\
132 	     insn = next_insn_same_sec(file, insn))
133 
134 static inline struct symbol *insn_call_dest(struct instruction *insn)
135 {
136 	if (insn->type == INSN_JUMP_DYNAMIC ||
137 	    insn->type == INSN_CALL_DYNAMIC)
138 		return NULL;
139 
140 	return insn->_call_dest;
141 }
142 
143 static inline struct reloc *insn_jump_table(struct instruction *insn)
144 {
145 	if (insn->type == INSN_JUMP_DYNAMIC ||
146 	    insn->type == INSN_CALL_DYNAMIC)
147 		return insn->_jump_table;
148 
149 	return NULL;
150 }
151 
152 static bool is_jump_table_jump(struct instruction *insn)
153 {
154 	struct alt_group *alt_group = insn->alt_group;
155 
156 	if (insn_jump_table(insn))
157 		return true;
158 
159 	/* Retpoline alternative for a jump table? */
160 	return alt_group && alt_group->orig_group &&
161 	       insn_jump_table(alt_group->orig_group->first_insn);
162 }
163 
164 static bool is_sibling_call(struct instruction *insn)
165 {
166 	/*
167 	 * Assume only STT_FUNC calls have jump-tables.
168 	 */
169 	if (insn_func(insn)) {
170 		/* An indirect jump is either a sibling call or a jump to a table. */
171 		if (insn->type == INSN_JUMP_DYNAMIC)
172 			return !is_jump_table_jump(insn);
173 	}
174 
175 	/* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
176 	return (is_static_jump(insn) && insn_call_dest(insn));
177 }
178 
179 /*
180  * This checks to see if the given function is a "noreturn" function.
181  *
182  * For global functions which are outside the scope of this object file, we
183  * have to keep a manual list of them.
184  *
185  * For local functions, we have to detect them manually by simply looking for
186  * the lack of a return instruction.
187  */
188 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
189 				int recursion)
190 {
191 	int i;
192 	struct instruction *insn;
193 	bool empty = true;
194 
195 #define NORETURN(func) __stringify(func),
196 	static const char * const global_noreturns[] = {
197 #include "noreturns.h"
198 	};
199 #undef NORETURN
200 
201 	if (!func)
202 		return false;
203 
204 	if (func->bind == STB_GLOBAL || func->bind == STB_WEAK)
205 		for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
206 			if (!strcmp(func->name, global_noreturns[i]))
207 				return true;
208 
209 	if (func->bind == STB_WEAK)
210 		return false;
211 
212 	if (!func->len)
213 		return false;
214 
215 	insn = find_insn(file, func->sec, func->offset);
216 	if (!insn || !insn_func(insn))
217 		return false;
218 
219 	func_for_each_insn(file, func, insn) {
220 		empty = false;
221 
222 		if (insn->type == INSN_RETURN)
223 			return false;
224 	}
225 
226 	if (empty)
227 		return false;
228 
229 	/*
230 	 * A function can have a sibling call instead of a return.  In that
231 	 * case, the function's dead-end status depends on whether the target
232 	 * of the sibling call returns.
233 	 */
234 	func_for_each_insn(file, func, insn) {
235 		if (is_sibling_call(insn)) {
236 			struct instruction *dest = insn->jump_dest;
237 
238 			if (!dest)
239 				/* sibling call to another file */
240 				return false;
241 
242 			/* local sibling call */
243 			if (recursion == 5) {
244 				/*
245 				 * Infinite recursion: two functions have
246 				 * sibling calls to each other.  This is a very
247 				 * rare case.  It means they aren't dead ends.
248 				 */
249 				return false;
250 			}
251 
252 			return __dead_end_function(file, insn_func(dest), recursion+1);
253 		}
254 	}
255 
256 	return true;
257 }
258 
259 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
260 {
261 	return __dead_end_function(file, func, 0);
262 }
263 
264 static void init_cfi_state(struct cfi_state *cfi)
265 {
266 	int i;
267 
268 	for (i = 0; i < CFI_NUM_REGS; i++) {
269 		cfi->regs[i].base = CFI_UNDEFINED;
270 		cfi->vals[i].base = CFI_UNDEFINED;
271 	}
272 	cfi->cfa.base = CFI_UNDEFINED;
273 	cfi->drap_reg = CFI_UNDEFINED;
274 	cfi->drap_offset = -1;
275 }
276 
277 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
278 			    struct section *sec)
279 {
280 	memset(state, 0, sizeof(*state));
281 	init_cfi_state(&state->cfi);
282 
283 	/*
284 	 * We need the full vmlinux for noinstr validation, otherwise we can
285 	 * not correctly determine insn_call_dest(insn)->sec (external symbols
286 	 * do not have a section).
287 	 */
288 	if (opts.link && opts.noinstr && sec)
289 		state->noinstr = sec->noinstr;
290 }
291 
292 static struct cfi_state *cfi_alloc(void)
293 {
294 	struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
295 	if (!cfi) {
296 		WARN("calloc failed");
297 		exit(1);
298 	}
299 	nr_cfi++;
300 	return cfi;
301 }
302 
303 static int cfi_bits;
304 static struct hlist_head *cfi_hash;
305 
306 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
307 {
308 	return memcmp((void *)cfi1 + sizeof(cfi1->hash),
309 		      (void *)cfi2 + sizeof(cfi2->hash),
310 		      sizeof(struct cfi_state) - sizeof(struct hlist_node));
311 }
312 
313 static inline u32 cfi_key(struct cfi_state *cfi)
314 {
315 	return jhash((void *)cfi + sizeof(cfi->hash),
316 		     sizeof(*cfi) - sizeof(cfi->hash), 0);
317 }
318 
319 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
320 {
321 	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
322 	struct cfi_state *obj;
323 
324 	hlist_for_each_entry(obj, head, hash) {
325 		if (!cficmp(cfi, obj)) {
326 			nr_cfi_cache++;
327 			return obj;
328 		}
329 	}
330 
331 	obj = cfi_alloc();
332 	*obj = *cfi;
333 	hlist_add_head(&obj->hash, head);
334 
335 	return obj;
336 }
337 
338 static void cfi_hash_add(struct cfi_state *cfi)
339 {
340 	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
341 
342 	hlist_add_head(&cfi->hash, head);
343 }
344 
345 static void *cfi_hash_alloc(unsigned long size)
346 {
347 	cfi_bits = max(10, ilog2(size));
348 	cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
349 			PROT_READ|PROT_WRITE,
350 			MAP_PRIVATE|MAP_ANON, -1, 0);
351 	if (cfi_hash == (void *)-1L) {
352 		WARN("mmap fail cfi_hash");
353 		cfi_hash = NULL;
354 	}  else if (opts.stats) {
355 		printf("cfi_bits: %d\n", cfi_bits);
356 	}
357 
358 	return cfi_hash;
359 }
360 
361 static unsigned long nr_insns;
362 static unsigned long nr_insns_visited;
363 
364 /*
365  * Call the arch-specific instruction decoder for all the instructions and add
366  * them to the global instruction list.
367  */
368 static int decode_instructions(struct objtool_file *file)
369 {
370 	struct section *sec;
371 	struct symbol *func;
372 	unsigned long offset;
373 	struct instruction *insn;
374 	int ret;
375 
376 	for_each_sec(file, sec) {
377 		struct instruction *insns = NULL;
378 		u8 prev_len = 0;
379 		u8 idx = 0;
380 
381 		if (!(sec->sh.sh_flags & SHF_EXECINSTR))
382 			continue;
383 
384 		if (strcmp(sec->name, ".altinstr_replacement") &&
385 		    strcmp(sec->name, ".altinstr_aux") &&
386 		    strncmp(sec->name, ".discard.", 9))
387 			sec->text = true;
388 
389 		if (!strcmp(sec->name, ".noinstr.text") ||
390 		    !strcmp(sec->name, ".entry.text") ||
391 		    !strcmp(sec->name, ".cpuidle.text") ||
392 		    !strncmp(sec->name, ".text..__x86.", 13))
393 			sec->noinstr = true;
394 
395 		/*
396 		 * .init.text code is ran before userspace and thus doesn't
397 		 * strictly need retpolines, except for modules which are
398 		 * loaded late, they very much do need retpoline in their
399 		 * .init.text
400 		 */
401 		if (!strcmp(sec->name, ".init.text") && !opts.module)
402 			sec->init = true;
403 
404 		for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
405 			if (!insns || idx == INSN_CHUNK_MAX) {
406 				insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
407 				if (!insns) {
408 					WARN("malloc failed");
409 					return -1;
410 				}
411 				idx = 0;
412 			} else {
413 				idx++;
414 			}
415 			insn = &insns[idx];
416 			insn->idx = idx;
417 
418 			INIT_LIST_HEAD(&insn->call_node);
419 			insn->sec = sec;
420 			insn->offset = offset;
421 			insn->prev_len = prev_len;
422 
423 			ret = arch_decode_instruction(file, sec, offset,
424 						      sec->sh.sh_size - offset,
425 						      insn);
426 			if (ret)
427 				return ret;
428 
429 			prev_len = insn->len;
430 
431 			/*
432 			 * By default, "ud2" is a dead end unless otherwise
433 			 * annotated, because GCC 7 inserts it for certain
434 			 * divide-by-zero cases.
435 			 */
436 			if (insn->type == INSN_BUG)
437 				insn->dead_end = true;
438 
439 			hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
440 			nr_insns++;
441 		}
442 
443 //		printf("%s: last chunk used: %d\n", sec->name, (int)idx);
444 
445 		sec_for_each_sym(sec, func) {
446 			if (func->type != STT_NOTYPE && func->type != STT_FUNC)
447 				continue;
448 
449 			if (func->offset == sec->sh.sh_size) {
450 				/* Heuristic: likely an "end" symbol */
451 				if (func->type == STT_NOTYPE)
452 					continue;
453 				WARN("%s(): STT_FUNC at end of section",
454 				     func->name);
455 				return -1;
456 			}
457 
458 			if (func->embedded_insn || func->alias != func)
459 				continue;
460 
461 			if (!find_insn(file, sec, func->offset)) {
462 				WARN("%s(): can't find starting instruction",
463 				     func->name);
464 				return -1;
465 			}
466 
467 			sym_for_each_insn(file, func, insn) {
468 				insn->sym = func;
469 				if (func->type == STT_FUNC &&
470 				    insn->type == INSN_ENDBR &&
471 				    list_empty(&insn->call_node)) {
472 					if (insn->offset == func->offset) {
473 						list_add_tail(&insn->call_node, &file->endbr_list);
474 						file->nr_endbr++;
475 					} else {
476 						file->nr_endbr_int++;
477 					}
478 				}
479 			}
480 		}
481 	}
482 
483 	if (opts.stats)
484 		printf("nr_insns: %lu\n", nr_insns);
485 
486 	return 0;
487 }
488 
489 /*
490  * Read the pv_ops[] .data table to find the static initialized values.
491  */
492 static int add_pv_ops(struct objtool_file *file, const char *symname)
493 {
494 	struct symbol *sym, *func;
495 	unsigned long off, end;
496 	struct reloc *reloc;
497 	int idx;
498 
499 	sym = find_symbol_by_name(file->elf, symname);
500 	if (!sym)
501 		return 0;
502 
503 	off = sym->offset;
504 	end = off + sym->len;
505 	for (;;) {
506 		reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
507 		if (!reloc)
508 			break;
509 
510 		func = reloc->sym;
511 		if (func->type == STT_SECTION)
512 			func = find_symbol_by_offset(reloc->sym->sec,
513 						     reloc_addend(reloc));
514 
515 		idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
516 
517 		objtool_pv_add(file, idx, func);
518 
519 		off = reloc_offset(reloc) + 1;
520 		if (off > end)
521 			break;
522 	}
523 
524 	return 0;
525 }
526 
527 /*
528  * Allocate and initialize file->pv_ops[].
529  */
530 static int init_pv_ops(struct objtool_file *file)
531 {
532 	static const char *pv_ops_tables[] = {
533 		"pv_ops",
534 		"xen_cpu_ops",
535 		"xen_irq_ops",
536 		"xen_mmu_ops",
537 		NULL,
538 	};
539 	const char *pv_ops;
540 	struct symbol *sym;
541 	int idx, nr;
542 
543 	if (!opts.noinstr)
544 		return 0;
545 
546 	file->pv_ops = NULL;
547 
548 	sym = find_symbol_by_name(file->elf, "pv_ops");
549 	if (!sym)
550 		return 0;
551 
552 	nr = sym->len / sizeof(unsigned long);
553 	file->pv_ops = calloc(sizeof(struct pv_state), nr);
554 	if (!file->pv_ops)
555 		return -1;
556 
557 	for (idx = 0; idx < nr; idx++)
558 		INIT_LIST_HEAD(&file->pv_ops[idx].targets);
559 
560 	for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
561 		add_pv_ops(file, pv_ops);
562 
563 	return 0;
564 }
565 
566 static struct instruction *find_last_insn(struct objtool_file *file,
567 					  struct section *sec)
568 {
569 	struct instruction *insn = NULL;
570 	unsigned int offset;
571 	unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
572 
573 	for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
574 		insn = find_insn(file, sec, offset);
575 
576 	return insn;
577 }
578 
579 /*
580  * Mark "ud2" instructions and manually annotated dead ends.
581  */
582 static int add_dead_ends(struct objtool_file *file)
583 {
584 	struct section *rsec;
585 	struct reloc *reloc;
586 	struct instruction *insn;
587 	s64 addend;
588 
589 	/*
590 	 * Check for manually annotated dead ends.
591 	 */
592 	rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
593 	if (!rsec)
594 		goto reachable;
595 
596 	for_each_reloc(rsec, reloc) {
597 
598 		if (reloc->sym->type != STT_SECTION) {
599 			WARN("unexpected relocation symbol type in %s", rsec->name);
600 			return -1;
601 		}
602 
603 		addend = reloc_addend(reloc);
604 
605 		insn = find_insn(file, reloc->sym->sec, addend);
606 		if (insn)
607 			insn = prev_insn_same_sec(file, insn);
608 		else if (addend == reloc->sym->sec->sh.sh_size) {
609 			insn = find_last_insn(file, reloc->sym->sec);
610 			if (!insn) {
611 				WARN("can't find unreachable insn at %s+0x%" PRIx64,
612 				     reloc->sym->sec->name, addend);
613 				return -1;
614 			}
615 		} else {
616 			WARN("can't find unreachable insn at %s+0x%" PRIx64,
617 			     reloc->sym->sec->name, addend);
618 			return -1;
619 		}
620 
621 		insn->dead_end = true;
622 	}
623 
624 reachable:
625 	/*
626 	 * These manually annotated reachable checks are needed for GCC 4.4,
627 	 * where the Linux unreachable() macro isn't supported.  In that case
628 	 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
629 	 * not a dead end.
630 	 */
631 	rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
632 	if (!rsec)
633 		return 0;
634 
635 	for_each_reloc(rsec, reloc) {
636 
637 		if (reloc->sym->type != STT_SECTION) {
638 			WARN("unexpected relocation symbol type in %s", rsec->name);
639 			return -1;
640 		}
641 
642 		addend = reloc_addend(reloc);
643 
644 		insn = find_insn(file, reloc->sym->sec, addend);
645 		if (insn)
646 			insn = prev_insn_same_sec(file, insn);
647 		else if (addend == reloc->sym->sec->sh.sh_size) {
648 			insn = find_last_insn(file, reloc->sym->sec);
649 			if (!insn) {
650 				WARN("can't find reachable insn at %s+0x%" PRIx64,
651 				     reloc->sym->sec->name, addend);
652 				return -1;
653 			}
654 		} else {
655 			WARN("can't find reachable insn at %s+0x%" PRIx64,
656 			     reloc->sym->sec->name, addend);
657 			return -1;
658 		}
659 
660 		insn->dead_end = false;
661 	}
662 
663 	return 0;
664 }
665 
666 static int create_static_call_sections(struct objtool_file *file)
667 {
668 	struct static_call_site *site;
669 	struct section *sec;
670 	struct instruction *insn;
671 	struct symbol *key_sym;
672 	char *key_name, *tmp;
673 	int idx;
674 
675 	sec = find_section_by_name(file->elf, ".static_call_sites");
676 	if (sec) {
677 		INIT_LIST_HEAD(&file->static_call_list);
678 		WARN("file already has .static_call_sites section, skipping");
679 		return 0;
680 	}
681 
682 	if (list_empty(&file->static_call_list))
683 		return 0;
684 
685 	idx = 0;
686 	list_for_each_entry(insn, &file->static_call_list, call_node)
687 		idx++;
688 
689 	sec = elf_create_section_pair(file->elf, ".static_call_sites",
690 				      sizeof(*site), idx, idx * 2);
691 	if (!sec)
692 		return -1;
693 
694 	/* Allow modules to modify the low bits of static_call_site::key */
695 	sec->sh.sh_flags |= SHF_WRITE;
696 
697 	idx = 0;
698 	list_for_each_entry(insn, &file->static_call_list, call_node) {
699 
700 		/* populate reloc for 'addr' */
701 		if (!elf_init_reloc_text_sym(file->elf, sec,
702 					     idx * sizeof(*site), idx * 2,
703 					     insn->sec, insn->offset))
704 			return -1;
705 
706 		/* find key symbol */
707 		key_name = strdup(insn_call_dest(insn)->name);
708 		if (!key_name) {
709 			perror("strdup");
710 			return -1;
711 		}
712 		if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
713 			    STATIC_CALL_TRAMP_PREFIX_LEN)) {
714 			WARN("static_call: trampoline name malformed: %s", key_name);
715 			free(key_name);
716 			return -1;
717 		}
718 		tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
719 		memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
720 
721 		key_sym = find_symbol_by_name(file->elf, tmp);
722 		if (!key_sym) {
723 			if (!opts.module) {
724 				WARN("static_call: can't find static_call_key symbol: %s", tmp);
725 				free(key_name);
726 				return -1;
727 			}
728 
729 			/*
730 			 * For modules(), the key might not be exported, which
731 			 * means the module can make static calls but isn't
732 			 * allowed to change them.
733 			 *
734 			 * In that case we temporarily set the key to be the
735 			 * trampoline address.  This is fixed up in
736 			 * static_call_add_module().
737 			 */
738 			key_sym = insn_call_dest(insn);
739 		}
740 		free(key_name);
741 
742 		/* populate reloc for 'key' */
743 		if (!elf_init_reloc_data_sym(file->elf, sec,
744 					     idx * sizeof(*site) + 4,
745 					     (idx * 2) + 1, key_sym,
746 					     is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
747 			return -1;
748 
749 		idx++;
750 	}
751 
752 	return 0;
753 }
754 
755 static int create_retpoline_sites_sections(struct objtool_file *file)
756 {
757 	struct instruction *insn;
758 	struct section *sec;
759 	int idx;
760 
761 	sec = find_section_by_name(file->elf, ".retpoline_sites");
762 	if (sec) {
763 		WARN("file already has .retpoline_sites, skipping");
764 		return 0;
765 	}
766 
767 	idx = 0;
768 	list_for_each_entry(insn, &file->retpoline_call_list, call_node)
769 		idx++;
770 
771 	if (!idx)
772 		return 0;
773 
774 	sec = elf_create_section_pair(file->elf, ".retpoline_sites",
775 				      sizeof(int), idx, idx);
776 	if (!sec)
777 		return -1;
778 
779 	idx = 0;
780 	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
781 
782 		if (!elf_init_reloc_text_sym(file->elf, sec,
783 					     idx * sizeof(int), idx,
784 					     insn->sec, insn->offset))
785 			return -1;
786 
787 		idx++;
788 	}
789 
790 	return 0;
791 }
792 
793 static int create_return_sites_sections(struct objtool_file *file)
794 {
795 	struct instruction *insn;
796 	struct section *sec;
797 	int idx;
798 
799 	sec = find_section_by_name(file->elf, ".return_sites");
800 	if (sec) {
801 		WARN("file already has .return_sites, skipping");
802 		return 0;
803 	}
804 
805 	idx = 0;
806 	list_for_each_entry(insn, &file->return_thunk_list, call_node)
807 		idx++;
808 
809 	if (!idx)
810 		return 0;
811 
812 	sec = elf_create_section_pair(file->elf, ".return_sites",
813 				      sizeof(int), idx, idx);
814 	if (!sec)
815 		return -1;
816 
817 	idx = 0;
818 	list_for_each_entry(insn, &file->return_thunk_list, call_node) {
819 
820 		if (!elf_init_reloc_text_sym(file->elf, sec,
821 					     idx * sizeof(int), idx,
822 					     insn->sec, insn->offset))
823 			return -1;
824 
825 		idx++;
826 	}
827 
828 	return 0;
829 }
830 
831 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
832 {
833 	struct instruction *insn;
834 	struct section *sec;
835 	int idx;
836 
837 	sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
838 	if (sec) {
839 		WARN("file already has .ibt_endbr_seal, skipping");
840 		return 0;
841 	}
842 
843 	idx = 0;
844 	list_for_each_entry(insn, &file->endbr_list, call_node)
845 		idx++;
846 
847 	if (opts.stats) {
848 		printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
849 		printf("ibt: ENDBR inside functions:  %d\n", file->nr_endbr_int);
850 		printf("ibt: superfluous ENDBR:       %d\n", idx);
851 	}
852 
853 	if (!idx)
854 		return 0;
855 
856 	sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
857 				      sizeof(int), idx, idx);
858 	if (!sec)
859 		return -1;
860 
861 	idx = 0;
862 	list_for_each_entry(insn, &file->endbr_list, call_node) {
863 
864 		int *site = (int *)sec->data->d_buf + idx;
865 		struct symbol *sym = insn->sym;
866 		*site = 0;
867 
868 		if (opts.module && sym && sym->type == STT_FUNC &&
869 		    insn->offset == sym->offset &&
870 		    (!strcmp(sym->name, "init_module") ||
871 		     !strcmp(sym->name, "cleanup_module")))
872 			WARN("%s(): not an indirect call target", sym->name);
873 
874 		if (!elf_init_reloc_text_sym(file->elf, sec,
875 					     idx * sizeof(int), idx,
876 					     insn->sec, insn->offset))
877 			return -1;
878 
879 		idx++;
880 	}
881 
882 	return 0;
883 }
884 
885 static int create_cfi_sections(struct objtool_file *file)
886 {
887 	struct section *sec;
888 	struct symbol *sym;
889 	int idx;
890 
891 	sec = find_section_by_name(file->elf, ".cfi_sites");
892 	if (sec) {
893 		INIT_LIST_HEAD(&file->call_list);
894 		WARN("file already has .cfi_sites section, skipping");
895 		return 0;
896 	}
897 
898 	idx = 0;
899 	for_each_sym(file, sym) {
900 		if (sym->type != STT_FUNC)
901 			continue;
902 
903 		if (strncmp(sym->name, "__cfi_", 6))
904 			continue;
905 
906 		idx++;
907 	}
908 
909 	sec = elf_create_section_pair(file->elf, ".cfi_sites",
910 				      sizeof(unsigned int), idx, idx);
911 	if (!sec)
912 		return -1;
913 
914 	idx = 0;
915 	for_each_sym(file, sym) {
916 		if (sym->type != STT_FUNC)
917 			continue;
918 
919 		if (strncmp(sym->name, "__cfi_", 6))
920 			continue;
921 
922 		if (!elf_init_reloc_text_sym(file->elf, sec,
923 					     idx * sizeof(unsigned int), idx,
924 					     sym->sec, sym->offset))
925 			return -1;
926 
927 		idx++;
928 	}
929 
930 	return 0;
931 }
932 
933 static int create_mcount_loc_sections(struct objtool_file *file)
934 {
935 	size_t addr_size = elf_addr_size(file->elf);
936 	struct instruction *insn;
937 	struct section *sec;
938 	int idx;
939 
940 	sec = find_section_by_name(file->elf, "__mcount_loc");
941 	if (sec) {
942 		INIT_LIST_HEAD(&file->mcount_loc_list);
943 		WARN("file already has __mcount_loc section, skipping");
944 		return 0;
945 	}
946 
947 	if (list_empty(&file->mcount_loc_list))
948 		return 0;
949 
950 	idx = 0;
951 	list_for_each_entry(insn, &file->mcount_loc_list, call_node)
952 		idx++;
953 
954 	sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
955 				      idx, idx);
956 	if (!sec)
957 		return -1;
958 
959 	sec->sh.sh_addralign = addr_size;
960 
961 	idx = 0;
962 	list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
963 
964 		struct reloc *reloc;
965 
966 		reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
967 					       insn->sec, insn->offset);
968 		if (!reloc)
969 			return -1;
970 
971 		set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
972 
973 		idx++;
974 	}
975 
976 	return 0;
977 }
978 
979 static int create_direct_call_sections(struct objtool_file *file)
980 {
981 	struct instruction *insn;
982 	struct section *sec;
983 	int idx;
984 
985 	sec = find_section_by_name(file->elf, ".call_sites");
986 	if (sec) {
987 		INIT_LIST_HEAD(&file->call_list);
988 		WARN("file already has .call_sites section, skipping");
989 		return 0;
990 	}
991 
992 	if (list_empty(&file->call_list))
993 		return 0;
994 
995 	idx = 0;
996 	list_for_each_entry(insn, &file->call_list, call_node)
997 		idx++;
998 
999 	sec = elf_create_section_pair(file->elf, ".call_sites",
1000 				      sizeof(unsigned int), idx, idx);
1001 	if (!sec)
1002 		return -1;
1003 
1004 	idx = 0;
1005 	list_for_each_entry(insn, &file->call_list, call_node) {
1006 
1007 		if (!elf_init_reloc_text_sym(file->elf, sec,
1008 					     idx * sizeof(unsigned int), idx,
1009 					     insn->sec, insn->offset))
1010 			return -1;
1011 
1012 		idx++;
1013 	}
1014 
1015 	return 0;
1016 }
1017 
1018 /*
1019  * Warnings shouldn't be reported for ignored functions.
1020  */
1021 static void add_ignores(struct objtool_file *file)
1022 {
1023 	struct instruction *insn;
1024 	struct section *rsec;
1025 	struct symbol *func;
1026 	struct reloc *reloc;
1027 
1028 	rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1029 	if (!rsec)
1030 		return;
1031 
1032 	for_each_reloc(rsec, reloc) {
1033 		switch (reloc->sym->type) {
1034 		case STT_FUNC:
1035 			func = reloc->sym;
1036 			break;
1037 
1038 		case STT_SECTION:
1039 			func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1040 			if (!func)
1041 				continue;
1042 			break;
1043 
1044 		default:
1045 			WARN("unexpected relocation symbol type in %s: %d",
1046 			     rsec->name, reloc->sym->type);
1047 			continue;
1048 		}
1049 
1050 		func_for_each_insn(file, func, insn)
1051 			insn->ignore = true;
1052 	}
1053 }
1054 
1055 /*
1056  * This is a whitelist of functions that is allowed to be called with AC set.
1057  * The list is meant to be minimal and only contains compiler instrumentation
1058  * ABI and a few functions used to implement *_{to,from}_user() functions.
1059  *
1060  * These functions must not directly change AC, but may PUSHF/POPF.
1061  */
1062 static const char *uaccess_safe_builtin[] = {
1063 	/* KASAN */
1064 	"kasan_report",
1065 	"kasan_check_range",
1066 	/* KASAN out-of-line */
1067 	"__asan_loadN_noabort",
1068 	"__asan_load1_noabort",
1069 	"__asan_load2_noabort",
1070 	"__asan_load4_noabort",
1071 	"__asan_load8_noabort",
1072 	"__asan_load16_noabort",
1073 	"__asan_storeN_noabort",
1074 	"__asan_store1_noabort",
1075 	"__asan_store2_noabort",
1076 	"__asan_store4_noabort",
1077 	"__asan_store8_noabort",
1078 	"__asan_store16_noabort",
1079 	"__kasan_check_read",
1080 	"__kasan_check_write",
1081 	/* KASAN in-line */
1082 	"__asan_report_load_n_noabort",
1083 	"__asan_report_load1_noabort",
1084 	"__asan_report_load2_noabort",
1085 	"__asan_report_load4_noabort",
1086 	"__asan_report_load8_noabort",
1087 	"__asan_report_load16_noabort",
1088 	"__asan_report_store_n_noabort",
1089 	"__asan_report_store1_noabort",
1090 	"__asan_report_store2_noabort",
1091 	"__asan_report_store4_noabort",
1092 	"__asan_report_store8_noabort",
1093 	"__asan_report_store16_noabort",
1094 	/* KCSAN */
1095 	"__kcsan_check_access",
1096 	"__kcsan_mb",
1097 	"__kcsan_wmb",
1098 	"__kcsan_rmb",
1099 	"__kcsan_release",
1100 	"kcsan_found_watchpoint",
1101 	"kcsan_setup_watchpoint",
1102 	"kcsan_check_scoped_accesses",
1103 	"kcsan_disable_current",
1104 	"kcsan_enable_current_nowarn",
1105 	/* KCSAN/TSAN */
1106 	"__tsan_func_entry",
1107 	"__tsan_func_exit",
1108 	"__tsan_read_range",
1109 	"__tsan_write_range",
1110 	"__tsan_read1",
1111 	"__tsan_read2",
1112 	"__tsan_read4",
1113 	"__tsan_read8",
1114 	"__tsan_read16",
1115 	"__tsan_write1",
1116 	"__tsan_write2",
1117 	"__tsan_write4",
1118 	"__tsan_write8",
1119 	"__tsan_write16",
1120 	"__tsan_read_write1",
1121 	"__tsan_read_write2",
1122 	"__tsan_read_write4",
1123 	"__tsan_read_write8",
1124 	"__tsan_read_write16",
1125 	"__tsan_volatile_read1",
1126 	"__tsan_volatile_read2",
1127 	"__tsan_volatile_read4",
1128 	"__tsan_volatile_read8",
1129 	"__tsan_volatile_read16",
1130 	"__tsan_volatile_write1",
1131 	"__tsan_volatile_write2",
1132 	"__tsan_volatile_write4",
1133 	"__tsan_volatile_write8",
1134 	"__tsan_volatile_write16",
1135 	"__tsan_atomic8_load",
1136 	"__tsan_atomic16_load",
1137 	"__tsan_atomic32_load",
1138 	"__tsan_atomic64_load",
1139 	"__tsan_atomic8_store",
1140 	"__tsan_atomic16_store",
1141 	"__tsan_atomic32_store",
1142 	"__tsan_atomic64_store",
1143 	"__tsan_atomic8_exchange",
1144 	"__tsan_atomic16_exchange",
1145 	"__tsan_atomic32_exchange",
1146 	"__tsan_atomic64_exchange",
1147 	"__tsan_atomic8_fetch_add",
1148 	"__tsan_atomic16_fetch_add",
1149 	"__tsan_atomic32_fetch_add",
1150 	"__tsan_atomic64_fetch_add",
1151 	"__tsan_atomic8_fetch_sub",
1152 	"__tsan_atomic16_fetch_sub",
1153 	"__tsan_atomic32_fetch_sub",
1154 	"__tsan_atomic64_fetch_sub",
1155 	"__tsan_atomic8_fetch_and",
1156 	"__tsan_atomic16_fetch_and",
1157 	"__tsan_atomic32_fetch_and",
1158 	"__tsan_atomic64_fetch_and",
1159 	"__tsan_atomic8_fetch_or",
1160 	"__tsan_atomic16_fetch_or",
1161 	"__tsan_atomic32_fetch_or",
1162 	"__tsan_atomic64_fetch_or",
1163 	"__tsan_atomic8_fetch_xor",
1164 	"__tsan_atomic16_fetch_xor",
1165 	"__tsan_atomic32_fetch_xor",
1166 	"__tsan_atomic64_fetch_xor",
1167 	"__tsan_atomic8_fetch_nand",
1168 	"__tsan_atomic16_fetch_nand",
1169 	"__tsan_atomic32_fetch_nand",
1170 	"__tsan_atomic64_fetch_nand",
1171 	"__tsan_atomic8_compare_exchange_strong",
1172 	"__tsan_atomic16_compare_exchange_strong",
1173 	"__tsan_atomic32_compare_exchange_strong",
1174 	"__tsan_atomic64_compare_exchange_strong",
1175 	"__tsan_atomic8_compare_exchange_weak",
1176 	"__tsan_atomic16_compare_exchange_weak",
1177 	"__tsan_atomic32_compare_exchange_weak",
1178 	"__tsan_atomic64_compare_exchange_weak",
1179 	"__tsan_atomic8_compare_exchange_val",
1180 	"__tsan_atomic16_compare_exchange_val",
1181 	"__tsan_atomic32_compare_exchange_val",
1182 	"__tsan_atomic64_compare_exchange_val",
1183 	"__tsan_atomic_thread_fence",
1184 	"__tsan_atomic_signal_fence",
1185 	"__tsan_unaligned_read16",
1186 	"__tsan_unaligned_write16",
1187 	/* KCOV */
1188 	"write_comp_data",
1189 	"check_kcov_mode",
1190 	"__sanitizer_cov_trace_pc",
1191 	"__sanitizer_cov_trace_const_cmp1",
1192 	"__sanitizer_cov_trace_const_cmp2",
1193 	"__sanitizer_cov_trace_const_cmp4",
1194 	"__sanitizer_cov_trace_const_cmp8",
1195 	"__sanitizer_cov_trace_cmp1",
1196 	"__sanitizer_cov_trace_cmp2",
1197 	"__sanitizer_cov_trace_cmp4",
1198 	"__sanitizer_cov_trace_cmp8",
1199 	"__sanitizer_cov_trace_switch",
1200 	/* KMSAN */
1201 	"kmsan_copy_to_user",
1202 	"kmsan_report",
1203 	"kmsan_unpoison_entry_regs",
1204 	"kmsan_unpoison_memory",
1205 	"__msan_chain_origin",
1206 	"__msan_get_context_state",
1207 	"__msan_instrument_asm_store",
1208 	"__msan_metadata_ptr_for_load_1",
1209 	"__msan_metadata_ptr_for_load_2",
1210 	"__msan_metadata_ptr_for_load_4",
1211 	"__msan_metadata_ptr_for_load_8",
1212 	"__msan_metadata_ptr_for_load_n",
1213 	"__msan_metadata_ptr_for_store_1",
1214 	"__msan_metadata_ptr_for_store_2",
1215 	"__msan_metadata_ptr_for_store_4",
1216 	"__msan_metadata_ptr_for_store_8",
1217 	"__msan_metadata_ptr_for_store_n",
1218 	"__msan_poison_alloca",
1219 	"__msan_warning",
1220 	/* UBSAN */
1221 	"ubsan_type_mismatch_common",
1222 	"__ubsan_handle_type_mismatch",
1223 	"__ubsan_handle_type_mismatch_v1",
1224 	"__ubsan_handle_shift_out_of_bounds",
1225 	"__ubsan_handle_load_invalid_value",
1226 	/* STACKLEAK */
1227 	"stackleak_track_stack",
1228 	/* misc */
1229 	"csum_partial_copy_generic",
1230 	"copy_mc_fragile",
1231 	"copy_mc_fragile_handle_tail",
1232 	"copy_mc_enhanced_fast_string",
1233 	"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1234 	"rep_stos_alternative",
1235 	"rep_movs_alternative",
1236 	"__copy_user_nocache",
1237 	NULL
1238 };
1239 
1240 static void add_uaccess_safe(struct objtool_file *file)
1241 {
1242 	struct symbol *func;
1243 	const char **name;
1244 
1245 	if (!opts.uaccess)
1246 		return;
1247 
1248 	for (name = uaccess_safe_builtin; *name; name++) {
1249 		func = find_symbol_by_name(file->elf, *name);
1250 		if (!func)
1251 			continue;
1252 
1253 		func->uaccess_safe = true;
1254 	}
1255 }
1256 
1257 /*
1258  * FIXME: For now, just ignore any alternatives which add retpolines.  This is
1259  * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1260  * But it at least allows objtool to understand the control flow *around* the
1261  * retpoline.
1262  */
1263 static int add_ignore_alternatives(struct objtool_file *file)
1264 {
1265 	struct section *rsec;
1266 	struct reloc *reloc;
1267 	struct instruction *insn;
1268 
1269 	rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1270 	if (!rsec)
1271 		return 0;
1272 
1273 	for_each_reloc(rsec, reloc) {
1274 		if (reloc->sym->type != STT_SECTION) {
1275 			WARN("unexpected relocation symbol type in %s", rsec->name);
1276 			return -1;
1277 		}
1278 
1279 		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
1280 		if (!insn) {
1281 			WARN("bad .discard.ignore_alts entry");
1282 			return -1;
1283 		}
1284 
1285 		insn->ignore_alts = true;
1286 	}
1287 
1288 	return 0;
1289 }
1290 
1291 /*
1292  * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1293  * will be added to the .retpoline_sites section.
1294  */
1295 __weak bool arch_is_retpoline(struct symbol *sym)
1296 {
1297 	return false;
1298 }
1299 
1300 /*
1301  * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1302  * will be added to the .return_sites section.
1303  */
1304 __weak bool arch_is_rethunk(struct symbol *sym)
1305 {
1306 	return false;
1307 }
1308 
1309 /*
1310  * Symbols that are embedded inside other instructions, because sometimes crazy
1311  * code exists. These are mostly ignored for validation purposes.
1312  */
1313 __weak bool arch_is_embedded_insn(struct symbol *sym)
1314 {
1315 	return false;
1316 }
1317 
1318 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1319 {
1320 	struct reloc *reloc;
1321 
1322 	if (insn->no_reloc)
1323 		return NULL;
1324 
1325 	if (!file)
1326 		return NULL;
1327 
1328 	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1329 					 insn->offset, insn->len);
1330 	if (!reloc) {
1331 		insn->no_reloc = 1;
1332 		return NULL;
1333 	}
1334 
1335 	return reloc;
1336 }
1337 
1338 static void remove_insn_ops(struct instruction *insn)
1339 {
1340 	struct stack_op *op, *next;
1341 
1342 	for (op = insn->stack_ops; op; op = next) {
1343 		next = op->next;
1344 		free(op);
1345 	}
1346 	insn->stack_ops = NULL;
1347 }
1348 
1349 static void annotate_call_site(struct objtool_file *file,
1350 			       struct instruction *insn, bool sibling)
1351 {
1352 	struct reloc *reloc = insn_reloc(file, insn);
1353 	struct symbol *sym = insn_call_dest(insn);
1354 
1355 	if (!sym)
1356 		sym = reloc->sym;
1357 
1358 	/*
1359 	 * Alternative replacement code is just template code which is
1360 	 * sometimes copied to the original instruction. For now, don't
1361 	 * annotate it. (In the future we might consider annotating the
1362 	 * original instruction if/when it ever makes sense to do so.)
1363 	 */
1364 	if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1365 		return;
1366 
1367 	if (sym->static_call_tramp) {
1368 		list_add_tail(&insn->call_node, &file->static_call_list);
1369 		return;
1370 	}
1371 
1372 	if (sym->retpoline_thunk) {
1373 		list_add_tail(&insn->call_node, &file->retpoline_call_list);
1374 		return;
1375 	}
1376 
1377 	/*
1378 	 * Many compilers cannot disable KCOV or sanitizer calls with a function
1379 	 * attribute so they need a little help, NOP out any such calls from
1380 	 * noinstr text.
1381 	 */
1382 	if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1383 		if (reloc)
1384 			set_reloc_type(file->elf, reloc, R_NONE);
1385 
1386 		elf_write_insn(file->elf, insn->sec,
1387 			       insn->offset, insn->len,
1388 			       sibling ? arch_ret_insn(insn->len)
1389 			               : arch_nop_insn(insn->len));
1390 
1391 		insn->type = sibling ? INSN_RETURN : INSN_NOP;
1392 
1393 		if (sibling) {
1394 			/*
1395 			 * We've replaced the tail-call JMP insn by two new
1396 			 * insn: RET; INT3, except we only have a single struct
1397 			 * insn here. Mark it retpoline_safe to avoid the SLS
1398 			 * warning, instead of adding another insn.
1399 			 */
1400 			insn->retpoline_safe = true;
1401 		}
1402 
1403 		return;
1404 	}
1405 
1406 	if (opts.mcount && sym->fentry) {
1407 		if (sibling)
1408 			WARN_INSN(insn, "tail call to __fentry__ !?!?");
1409 		if (opts.mnop) {
1410 			if (reloc)
1411 				set_reloc_type(file->elf, reloc, R_NONE);
1412 
1413 			elf_write_insn(file->elf, insn->sec,
1414 				       insn->offset, insn->len,
1415 				       arch_nop_insn(insn->len));
1416 
1417 			insn->type = INSN_NOP;
1418 		}
1419 
1420 		list_add_tail(&insn->call_node, &file->mcount_loc_list);
1421 		return;
1422 	}
1423 
1424 	if (insn->type == INSN_CALL && !insn->sec->init)
1425 		list_add_tail(&insn->call_node, &file->call_list);
1426 
1427 	if (!sibling && dead_end_function(file, sym))
1428 		insn->dead_end = true;
1429 }
1430 
1431 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1432 			  struct symbol *dest, bool sibling)
1433 {
1434 	insn->_call_dest = dest;
1435 	if (!dest)
1436 		return;
1437 
1438 	/*
1439 	 * Whatever stack impact regular CALLs have, should be undone
1440 	 * by the RETURN of the called function.
1441 	 *
1442 	 * Annotated intra-function calls retain the stack_ops but
1443 	 * are converted to JUMP, see read_intra_function_calls().
1444 	 */
1445 	remove_insn_ops(insn);
1446 
1447 	annotate_call_site(file, insn, sibling);
1448 }
1449 
1450 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1451 {
1452 	/*
1453 	 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1454 	 * so convert them accordingly.
1455 	 */
1456 	switch (insn->type) {
1457 	case INSN_CALL:
1458 		insn->type = INSN_CALL_DYNAMIC;
1459 		break;
1460 	case INSN_JUMP_UNCONDITIONAL:
1461 		insn->type = INSN_JUMP_DYNAMIC;
1462 		break;
1463 	case INSN_JUMP_CONDITIONAL:
1464 		insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1465 		break;
1466 	default:
1467 		return;
1468 	}
1469 
1470 	insn->retpoline_safe = true;
1471 
1472 	/*
1473 	 * Whatever stack impact regular CALLs have, should be undone
1474 	 * by the RETURN of the called function.
1475 	 *
1476 	 * Annotated intra-function calls retain the stack_ops but
1477 	 * are converted to JUMP, see read_intra_function_calls().
1478 	 */
1479 	remove_insn_ops(insn);
1480 
1481 	annotate_call_site(file, insn, false);
1482 }
1483 
1484 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1485 {
1486 	/*
1487 	 * Return thunk tail calls are really just returns in disguise,
1488 	 * so convert them accordingly.
1489 	 */
1490 	insn->type = INSN_RETURN;
1491 	insn->retpoline_safe = true;
1492 
1493 	if (add)
1494 		list_add_tail(&insn->call_node, &file->return_thunk_list);
1495 }
1496 
1497 static bool is_first_func_insn(struct objtool_file *file,
1498 			       struct instruction *insn, struct symbol *sym)
1499 {
1500 	if (insn->offset == sym->offset)
1501 		return true;
1502 
1503 	/* Allow direct CALL/JMP past ENDBR */
1504 	if (opts.ibt) {
1505 		struct instruction *prev = prev_insn_same_sym(file, insn);
1506 
1507 		if (prev && prev->type == INSN_ENDBR &&
1508 		    insn->offset == sym->offset + prev->len)
1509 			return true;
1510 	}
1511 
1512 	return false;
1513 }
1514 
1515 /*
1516  * A sibling call is a tail-call to another symbol -- to differentiate from a
1517  * recursive tail-call which is to the same symbol.
1518  */
1519 static bool jump_is_sibling_call(struct objtool_file *file,
1520 				 struct instruction *from, struct instruction *to)
1521 {
1522 	struct symbol *fs = from->sym;
1523 	struct symbol *ts = to->sym;
1524 
1525 	/* Not a sibling call if from/to a symbol hole */
1526 	if (!fs || !ts)
1527 		return false;
1528 
1529 	/* Not a sibling call if not targeting the start of a symbol. */
1530 	if (!is_first_func_insn(file, to, ts))
1531 		return false;
1532 
1533 	/* Disallow sibling calls into STT_NOTYPE */
1534 	if (ts->type == STT_NOTYPE)
1535 		return false;
1536 
1537 	/* Must not be self to be a sibling */
1538 	return fs->pfunc != ts->pfunc;
1539 }
1540 
1541 /*
1542  * Find the destination instructions for all jumps.
1543  */
1544 static int add_jump_destinations(struct objtool_file *file)
1545 {
1546 	struct instruction *insn, *jump_dest;
1547 	struct reloc *reloc;
1548 	struct section *dest_sec;
1549 	unsigned long dest_off;
1550 
1551 	for_each_insn(file, insn) {
1552 		if (insn->jump_dest) {
1553 			/*
1554 			 * handle_group_alt() may have previously set
1555 			 * 'jump_dest' for some alternatives.
1556 			 */
1557 			continue;
1558 		}
1559 		if (!is_static_jump(insn))
1560 			continue;
1561 
1562 		reloc = insn_reloc(file, insn);
1563 		if (!reloc) {
1564 			dest_sec = insn->sec;
1565 			dest_off = arch_jump_destination(insn);
1566 		} else if (reloc->sym->type == STT_SECTION) {
1567 			dest_sec = reloc->sym->sec;
1568 			dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1569 		} else if (reloc->sym->retpoline_thunk) {
1570 			add_retpoline_call(file, insn);
1571 			continue;
1572 		} else if (reloc->sym->return_thunk) {
1573 			add_return_call(file, insn, true);
1574 			continue;
1575 		} else if (insn_func(insn)) {
1576 			/*
1577 			 * External sibling call or internal sibling call with
1578 			 * STT_FUNC reloc.
1579 			 */
1580 			add_call_dest(file, insn, reloc->sym, true);
1581 			continue;
1582 		} else if (reloc->sym->sec->idx) {
1583 			dest_sec = reloc->sym->sec;
1584 			dest_off = reloc->sym->sym.st_value +
1585 				   arch_dest_reloc_offset(reloc_addend(reloc));
1586 		} else {
1587 			/* non-func asm code jumping to another file */
1588 			continue;
1589 		}
1590 
1591 		jump_dest = find_insn(file, dest_sec, dest_off);
1592 		if (!jump_dest) {
1593 			struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1594 
1595 			/*
1596 			 * This is a special case for retbleed_untrain_ret().
1597 			 * It jumps to __x86_return_thunk(), but objtool
1598 			 * can't find the thunk's starting RET
1599 			 * instruction, because the RET is also in the
1600 			 * middle of another instruction.  Objtool only
1601 			 * knows about the outer instruction.
1602 			 */
1603 			if (sym && sym->embedded_insn) {
1604 				add_return_call(file, insn, false);
1605 				continue;
1606 			}
1607 
1608 			WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1609 				  dest_sec->name, dest_off);
1610 			return -1;
1611 		}
1612 
1613 		/*
1614 		 * An intra-TU jump in retpoline.o might not have a relocation
1615 		 * for its jump dest, in which case the above
1616 		 * add_{retpoline,return}_call() didn't happen.
1617 		 */
1618 		if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1619 			if (jump_dest->sym->retpoline_thunk) {
1620 				add_retpoline_call(file, insn);
1621 				continue;
1622 			}
1623 			if (jump_dest->sym->return_thunk) {
1624 				add_return_call(file, insn, true);
1625 				continue;
1626 			}
1627 		}
1628 
1629 		/*
1630 		 * Cross-function jump.
1631 		 */
1632 		if (insn_func(insn) && insn_func(jump_dest) &&
1633 		    insn_func(insn) != insn_func(jump_dest)) {
1634 
1635 			/*
1636 			 * For GCC 8+, create parent/child links for any cold
1637 			 * subfunctions.  This is _mostly_ redundant with a
1638 			 * similar initialization in read_symbols().
1639 			 *
1640 			 * If a function has aliases, we want the *first* such
1641 			 * function in the symbol table to be the subfunction's
1642 			 * parent.  In that case we overwrite the
1643 			 * initialization done in read_symbols().
1644 			 *
1645 			 * However this code can't completely replace the
1646 			 * read_symbols() code because this doesn't detect the
1647 			 * case where the parent function's only reference to a
1648 			 * subfunction is through a jump table.
1649 			 */
1650 			if (!strstr(insn_func(insn)->name, ".cold") &&
1651 			    strstr(insn_func(jump_dest)->name, ".cold")) {
1652 				insn_func(insn)->cfunc = insn_func(jump_dest);
1653 				insn_func(jump_dest)->pfunc = insn_func(insn);
1654 			}
1655 		}
1656 
1657 		if (jump_is_sibling_call(file, insn, jump_dest)) {
1658 			/*
1659 			 * Internal sibling call without reloc or with
1660 			 * STT_SECTION reloc.
1661 			 */
1662 			add_call_dest(file, insn, insn_func(jump_dest), true);
1663 			continue;
1664 		}
1665 
1666 		insn->jump_dest = jump_dest;
1667 	}
1668 
1669 	return 0;
1670 }
1671 
1672 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1673 {
1674 	struct symbol *call_dest;
1675 
1676 	call_dest = find_func_by_offset(sec, offset);
1677 	if (!call_dest)
1678 		call_dest = find_symbol_by_offset(sec, offset);
1679 
1680 	return call_dest;
1681 }
1682 
1683 /*
1684  * Find the destination instructions for all calls.
1685  */
1686 static int add_call_destinations(struct objtool_file *file)
1687 {
1688 	struct instruction *insn;
1689 	unsigned long dest_off;
1690 	struct symbol *dest;
1691 	struct reloc *reloc;
1692 
1693 	for_each_insn(file, insn) {
1694 		if (insn->type != INSN_CALL)
1695 			continue;
1696 
1697 		reloc = insn_reloc(file, insn);
1698 		if (!reloc) {
1699 			dest_off = arch_jump_destination(insn);
1700 			dest = find_call_destination(insn->sec, dest_off);
1701 
1702 			add_call_dest(file, insn, dest, false);
1703 
1704 			if (insn->ignore)
1705 				continue;
1706 
1707 			if (!insn_call_dest(insn)) {
1708 				WARN_INSN(insn, "unannotated intra-function call");
1709 				return -1;
1710 			}
1711 
1712 			if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1713 				WARN_INSN(insn, "unsupported call to non-function");
1714 				return -1;
1715 			}
1716 
1717 		} else if (reloc->sym->type == STT_SECTION) {
1718 			dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1719 			dest = find_call_destination(reloc->sym->sec, dest_off);
1720 			if (!dest) {
1721 				WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1722 					  reloc->sym->sec->name, dest_off);
1723 				return -1;
1724 			}
1725 
1726 			add_call_dest(file, insn, dest, false);
1727 
1728 		} else if (reloc->sym->retpoline_thunk) {
1729 			add_retpoline_call(file, insn);
1730 
1731 		} else
1732 			add_call_dest(file, insn, reloc->sym, false);
1733 	}
1734 
1735 	return 0;
1736 }
1737 
1738 /*
1739  * The .alternatives section requires some extra special care over and above
1740  * other special sections because alternatives are patched in place.
1741  */
1742 static int handle_group_alt(struct objtool_file *file,
1743 			    struct special_alt *special_alt,
1744 			    struct instruction *orig_insn,
1745 			    struct instruction **new_insn)
1746 {
1747 	struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1748 	struct alt_group *orig_alt_group, *new_alt_group;
1749 	unsigned long dest_off;
1750 
1751 	orig_alt_group = orig_insn->alt_group;
1752 	if (!orig_alt_group) {
1753 		struct instruction *last_orig_insn = NULL;
1754 
1755 		orig_alt_group = malloc(sizeof(*orig_alt_group));
1756 		if (!orig_alt_group) {
1757 			WARN("malloc failed");
1758 			return -1;
1759 		}
1760 		orig_alt_group->cfi = calloc(special_alt->orig_len,
1761 					     sizeof(struct cfi_state *));
1762 		if (!orig_alt_group->cfi) {
1763 			WARN("calloc failed");
1764 			return -1;
1765 		}
1766 
1767 		insn = orig_insn;
1768 		sec_for_each_insn_from(file, insn) {
1769 			if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1770 				break;
1771 
1772 			insn->alt_group = orig_alt_group;
1773 			last_orig_insn = insn;
1774 		}
1775 		orig_alt_group->orig_group = NULL;
1776 		orig_alt_group->first_insn = orig_insn;
1777 		orig_alt_group->last_insn = last_orig_insn;
1778 		orig_alt_group->nop = NULL;
1779 	} else {
1780 		if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1781 		    orig_alt_group->first_insn->offset != special_alt->orig_len) {
1782 			WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1783 				  orig_alt_group->last_insn->offset +
1784 				  orig_alt_group->last_insn->len -
1785 				  orig_alt_group->first_insn->offset,
1786 				  special_alt->orig_len);
1787 			return -1;
1788 		}
1789 	}
1790 
1791 	new_alt_group = malloc(sizeof(*new_alt_group));
1792 	if (!new_alt_group) {
1793 		WARN("malloc failed");
1794 		return -1;
1795 	}
1796 
1797 	if (special_alt->new_len < special_alt->orig_len) {
1798 		/*
1799 		 * Insert a fake nop at the end to make the replacement
1800 		 * alt_group the same size as the original.  This is needed to
1801 		 * allow propagate_alt_cfi() to do its magic.  When the last
1802 		 * instruction affects the stack, the instruction after it (the
1803 		 * nop) will propagate the new state to the shared CFI array.
1804 		 */
1805 		nop = malloc(sizeof(*nop));
1806 		if (!nop) {
1807 			WARN("malloc failed");
1808 			return -1;
1809 		}
1810 		memset(nop, 0, sizeof(*nop));
1811 
1812 		nop->sec = special_alt->new_sec;
1813 		nop->offset = special_alt->new_off + special_alt->new_len;
1814 		nop->len = special_alt->orig_len - special_alt->new_len;
1815 		nop->type = INSN_NOP;
1816 		nop->sym = orig_insn->sym;
1817 		nop->alt_group = new_alt_group;
1818 		nop->ignore = orig_insn->ignore_alts;
1819 	}
1820 
1821 	if (!special_alt->new_len) {
1822 		*new_insn = nop;
1823 		goto end;
1824 	}
1825 
1826 	insn = *new_insn;
1827 	sec_for_each_insn_from(file, insn) {
1828 		struct reloc *alt_reloc;
1829 
1830 		if (insn->offset >= special_alt->new_off + special_alt->new_len)
1831 			break;
1832 
1833 		last_new_insn = insn;
1834 
1835 		insn->ignore = orig_insn->ignore_alts;
1836 		insn->sym = orig_insn->sym;
1837 		insn->alt_group = new_alt_group;
1838 
1839 		/*
1840 		 * Since alternative replacement code is copy/pasted by the
1841 		 * kernel after applying relocations, generally such code can't
1842 		 * have relative-address relocation references to outside the
1843 		 * .altinstr_replacement section, unless the arch's
1844 		 * alternatives code can adjust the relative offsets
1845 		 * accordingly.
1846 		 */
1847 		alt_reloc = insn_reloc(file, insn);
1848 		if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1849 		    !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1850 
1851 			WARN_INSN(insn, "unsupported relocation in alternatives section");
1852 			return -1;
1853 		}
1854 
1855 		if (!is_static_jump(insn))
1856 			continue;
1857 
1858 		if (!insn->immediate)
1859 			continue;
1860 
1861 		dest_off = arch_jump_destination(insn);
1862 		if (dest_off == special_alt->new_off + special_alt->new_len) {
1863 			insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1864 			if (!insn->jump_dest) {
1865 				WARN_INSN(insn, "can't find alternative jump destination");
1866 				return -1;
1867 			}
1868 		}
1869 	}
1870 
1871 	if (!last_new_insn) {
1872 		WARN_FUNC("can't find last new alternative instruction",
1873 			  special_alt->new_sec, special_alt->new_off);
1874 		return -1;
1875 	}
1876 
1877 end:
1878 	new_alt_group->orig_group = orig_alt_group;
1879 	new_alt_group->first_insn = *new_insn;
1880 	new_alt_group->last_insn = last_new_insn;
1881 	new_alt_group->nop = nop;
1882 	new_alt_group->cfi = orig_alt_group->cfi;
1883 	return 0;
1884 }
1885 
1886 /*
1887  * A jump table entry can either convert a nop to a jump or a jump to a nop.
1888  * If the original instruction is a jump, make the alt entry an effective nop
1889  * by just skipping the original instruction.
1890  */
1891 static int handle_jump_alt(struct objtool_file *file,
1892 			   struct special_alt *special_alt,
1893 			   struct instruction *orig_insn,
1894 			   struct instruction **new_insn)
1895 {
1896 	if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1897 	    orig_insn->type != INSN_NOP) {
1898 
1899 		WARN_INSN(orig_insn, "unsupported instruction at jump label");
1900 		return -1;
1901 	}
1902 
1903 	if (opts.hack_jump_label && special_alt->key_addend & 2) {
1904 		struct reloc *reloc = insn_reloc(file, orig_insn);
1905 
1906 		if (reloc)
1907 			set_reloc_type(file->elf, reloc, R_NONE);
1908 		elf_write_insn(file->elf, orig_insn->sec,
1909 			       orig_insn->offset, orig_insn->len,
1910 			       arch_nop_insn(orig_insn->len));
1911 		orig_insn->type = INSN_NOP;
1912 	}
1913 
1914 	if (orig_insn->type == INSN_NOP) {
1915 		if (orig_insn->len == 2)
1916 			file->jl_nop_short++;
1917 		else
1918 			file->jl_nop_long++;
1919 
1920 		return 0;
1921 	}
1922 
1923 	if (orig_insn->len == 2)
1924 		file->jl_short++;
1925 	else
1926 		file->jl_long++;
1927 
1928 	*new_insn = next_insn_same_sec(file, orig_insn);
1929 	return 0;
1930 }
1931 
1932 /*
1933  * Read all the special sections which have alternate instructions which can be
1934  * patched in or redirected to at runtime.  Each instruction having alternate
1935  * instruction(s) has them added to its insn->alts list, which will be
1936  * traversed in validate_branch().
1937  */
1938 static int add_special_section_alts(struct objtool_file *file)
1939 {
1940 	struct list_head special_alts;
1941 	struct instruction *orig_insn, *new_insn;
1942 	struct special_alt *special_alt, *tmp;
1943 	struct alternative *alt;
1944 	int ret;
1945 
1946 	ret = special_get_alts(file->elf, &special_alts);
1947 	if (ret)
1948 		return ret;
1949 
1950 	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1951 
1952 		orig_insn = find_insn(file, special_alt->orig_sec,
1953 				      special_alt->orig_off);
1954 		if (!orig_insn) {
1955 			WARN_FUNC("special: can't find orig instruction",
1956 				  special_alt->orig_sec, special_alt->orig_off);
1957 			ret = -1;
1958 			goto out;
1959 		}
1960 
1961 		new_insn = NULL;
1962 		if (!special_alt->group || special_alt->new_len) {
1963 			new_insn = find_insn(file, special_alt->new_sec,
1964 					     special_alt->new_off);
1965 			if (!new_insn) {
1966 				WARN_FUNC("special: can't find new instruction",
1967 					  special_alt->new_sec,
1968 					  special_alt->new_off);
1969 				ret = -1;
1970 				goto out;
1971 			}
1972 		}
1973 
1974 		if (special_alt->group) {
1975 			if (!special_alt->orig_len) {
1976 				WARN_INSN(orig_insn, "empty alternative entry");
1977 				continue;
1978 			}
1979 
1980 			ret = handle_group_alt(file, special_alt, orig_insn,
1981 					       &new_insn);
1982 			if (ret)
1983 				goto out;
1984 		} else if (special_alt->jump_or_nop) {
1985 			ret = handle_jump_alt(file, special_alt, orig_insn,
1986 					      &new_insn);
1987 			if (ret)
1988 				goto out;
1989 		}
1990 
1991 		alt = malloc(sizeof(*alt));
1992 		if (!alt) {
1993 			WARN("malloc failed");
1994 			ret = -1;
1995 			goto out;
1996 		}
1997 
1998 		alt->insn = new_insn;
1999 		alt->skip_orig = special_alt->skip_orig;
2000 		orig_insn->ignore_alts |= special_alt->skip_alt;
2001 		alt->next = orig_insn->alts;
2002 		orig_insn->alts = alt;
2003 
2004 		list_del(&special_alt->list);
2005 		free(special_alt);
2006 	}
2007 
2008 	if (opts.stats) {
2009 		printf("jl\\\tNOP\tJMP\n");
2010 		printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2011 		printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2012 	}
2013 
2014 out:
2015 	return ret;
2016 }
2017 
2018 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2019 			  struct reloc *next_table)
2020 {
2021 	struct symbol *pfunc = insn_func(insn)->pfunc;
2022 	struct reloc *table = insn_jump_table(insn);
2023 	struct instruction *dest_insn;
2024 	unsigned int prev_offset = 0;
2025 	struct reloc *reloc = table;
2026 	struct alternative *alt;
2027 
2028 	/*
2029 	 * Each @reloc is a switch table relocation which points to the target
2030 	 * instruction.
2031 	 */
2032 	for_each_reloc_from(table->sec, reloc) {
2033 
2034 		/* Check for the end of the table: */
2035 		if (reloc != table && reloc == next_table)
2036 			break;
2037 
2038 		/* Make sure the table entries are consecutive: */
2039 		if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2040 			break;
2041 
2042 		/* Detect function pointers from contiguous objects: */
2043 		if (reloc->sym->sec == pfunc->sec &&
2044 		    reloc_addend(reloc) == pfunc->offset)
2045 			break;
2046 
2047 		dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2048 		if (!dest_insn)
2049 			break;
2050 
2051 		/* Make sure the destination is in the same function: */
2052 		if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2053 			break;
2054 
2055 		alt = malloc(sizeof(*alt));
2056 		if (!alt) {
2057 			WARN("malloc failed");
2058 			return -1;
2059 		}
2060 
2061 		alt->insn = dest_insn;
2062 		alt->next = insn->alts;
2063 		insn->alts = alt;
2064 		prev_offset = reloc_offset(reloc);
2065 	}
2066 
2067 	if (!prev_offset) {
2068 		WARN_INSN(insn, "can't find switch jump table");
2069 		return -1;
2070 	}
2071 
2072 	return 0;
2073 }
2074 
2075 /*
2076  * find_jump_table() - Given a dynamic jump, find the switch jump table
2077  * associated with it.
2078  */
2079 static struct reloc *find_jump_table(struct objtool_file *file,
2080 				      struct symbol *func,
2081 				      struct instruction *insn)
2082 {
2083 	struct reloc *table_reloc;
2084 	struct instruction *dest_insn, *orig_insn = insn;
2085 
2086 	/*
2087 	 * Backward search using the @first_jump_src links, these help avoid
2088 	 * much of the 'in between' code. Which avoids us getting confused by
2089 	 * it.
2090 	 */
2091 	for (;
2092 	     insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2093 	     insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2094 
2095 		if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2096 			break;
2097 
2098 		/* allow small jumps within the range */
2099 		if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2100 		    insn->jump_dest &&
2101 		    (insn->jump_dest->offset <= insn->offset ||
2102 		     insn->jump_dest->offset > orig_insn->offset))
2103 		    break;
2104 
2105 		table_reloc = arch_find_switch_table(file, insn);
2106 		if (!table_reloc)
2107 			continue;
2108 		dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc));
2109 		if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2110 			continue;
2111 
2112 		return table_reloc;
2113 	}
2114 
2115 	return NULL;
2116 }
2117 
2118 /*
2119  * First pass: Mark the head of each jump table so that in the next pass,
2120  * we know when a given jump table ends and the next one starts.
2121  */
2122 static void mark_func_jump_tables(struct objtool_file *file,
2123 				    struct symbol *func)
2124 {
2125 	struct instruction *insn, *last = NULL;
2126 	struct reloc *reloc;
2127 
2128 	func_for_each_insn(file, func, insn) {
2129 		if (!last)
2130 			last = insn;
2131 
2132 		/*
2133 		 * Store back-pointers for unconditional forward jumps such
2134 		 * that find_jump_table() can back-track using those and
2135 		 * avoid some potentially confusing code.
2136 		 */
2137 		if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2138 		    insn->offset > last->offset &&
2139 		    insn->jump_dest->offset > insn->offset &&
2140 		    !insn->jump_dest->first_jump_src) {
2141 
2142 			insn->jump_dest->first_jump_src = insn;
2143 			last = insn->jump_dest;
2144 		}
2145 
2146 		if (insn->type != INSN_JUMP_DYNAMIC)
2147 			continue;
2148 
2149 		reloc = find_jump_table(file, func, insn);
2150 		if (reloc)
2151 			insn->_jump_table = reloc;
2152 	}
2153 }
2154 
2155 static int add_func_jump_tables(struct objtool_file *file,
2156 				  struct symbol *func)
2157 {
2158 	struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2159 	int ret = 0;
2160 
2161 	func_for_each_insn(file, func, insn) {
2162 		if (!insn_jump_table(insn))
2163 			continue;
2164 
2165 		if (!insn_t1) {
2166 			insn_t1 = insn;
2167 			continue;
2168 		}
2169 
2170 		insn_t2 = insn;
2171 
2172 		ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2173 		if (ret)
2174 			return ret;
2175 
2176 		insn_t1 = insn_t2;
2177 	}
2178 
2179 	if (insn_t1)
2180 		ret = add_jump_table(file, insn_t1, NULL);
2181 
2182 	return ret;
2183 }
2184 
2185 /*
2186  * For some switch statements, gcc generates a jump table in the .rodata
2187  * section which contains a list of addresses within the function to jump to.
2188  * This finds these jump tables and adds them to the insn->alts lists.
2189  */
2190 static int add_jump_table_alts(struct objtool_file *file)
2191 {
2192 	struct symbol *func;
2193 	int ret;
2194 
2195 	if (!file->rodata)
2196 		return 0;
2197 
2198 	for_each_sym(file, func) {
2199 		if (func->type != STT_FUNC)
2200 			continue;
2201 
2202 		mark_func_jump_tables(file, func);
2203 		ret = add_func_jump_tables(file, func);
2204 		if (ret)
2205 			return ret;
2206 	}
2207 
2208 	return 0;
2209 }
2210 
2211 static void set_func_state(struct cfi_state *state)
2212 {
2213 	state->cfa = initial_func_cfi.cfa;
2214 	memcpy(&state->regs, &initial_func_cfi.regs,
2215 	       CFI_NUM_REGS * sizeof(struct cfi_reg));
2216 	state->stack_size = initial_func_cfi.cfa.offset;
2217 	state->type = UNWIND_HINT_TYPE_CALL;
2218 }
2219 
2220 static int read_unwind_hints(struct objtool_file *file)
2221 {
2222 	struct cfi_state cfi = init_cfi;
2223 	struct section *sec;
2224 	struct unwind_hint *hint;
2225 	struct instruction *insn;
2226 	struct reloc *reloc;
2227 	int i;
2228 
2229 	sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2230 	if (!sec)
2231 		return 0;
2232 
2233 	if (!sec->rsec) {
2234 		WARN("missing .rela.discard.unwind_hints section");
2235 		return -1;
2236 	}
2237 
2238 	if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2239 		WARN("struct unwind_hint size mismatch");
2240 		return -1;
2241 	}
2242 
2243 	file->hints = true;
2244 
2245 	for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2246 		hint = (struct unwind_hint *)sec->data->d_buf + i;
2247 
2248 		reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2249 		if (!reloc) {
2250 			WARN("can't find reloc for unwind_hints[%d]", i);
2251 			return -1;
2252 		}
2253 
2254 		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2255 		if (!insn) {
2256 			WARN("can't find insn for unwind_hints[%d]", i);
2257 			return -1;
2258 		}
2259 
2260 		insn->hint = true;
2261 
2262 		if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2263 			insn->cfi = &force_undefined_cfi;
2264 			continue;
2265 		}
2266 
2267 		if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2268 			insn->hint = false;
2269 			insn->save = true;
2270 			continue;
2271 		}
2272 
2273 		if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2274 			insn->restore = true;
2275 			continue;
2276 		}
2277 
2278 		if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2279 			struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2280 
2281 			if (sym && sym->bind == STB_GLOBAL) {
2282 				if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2283 					WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2284 				}
2285 			}
2286 		}
2287 
2288 		if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2289 			insn->cfi = &func_cfi;
2290 			continue;
2291 		}
2292 
2293 		if (insn->cfi)
2294 			cfi = *(insn->cfi);
2295 
2296 		if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2297 			WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2298 			return -1;
2299 		}
2300 
2301 		cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2302 		cfi.type = hint->type;
2303 		cfi.signal = hint->signal;
2304 
2305 		insn->cfi = cfi_hash_find_or_add(&cfi);
2306 	}
2307 
2308 	return 0;
2309 }
2310 
2311 static int read_noendbr_hints(struct objtool_file *file)
2312 {
2313 	struct instruction *insn;
2314 	struct section *rsec;
2315 	struct reloc *reloc;
2316 
2317 	rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2318 	if (!rsec)
2319 		return 0;
2320 
2321 	for_each_reloc(rsec, reloc) {
2322 		insn = find_insn(file, reloc->sym->sec,
2323 				 reloc->sym->offset + reloc_addend(reloc));
2324 		if (!insn) {
2325 			WARN("bad .discard.noendbr entry");
2326 			return -1;
2327 		}
2328 
2329 		insn->noendbr = 1;
2330 	}
2331 
2332 	return 0;
2333 }
2334 
2335 static int read_retpoline_hints(struct objtool_file *file)
2336 {
2337 	struct section *rsec;
2338 	struct instruction *insn;
2339 	struct reloc *reloc;
2340 
2341 	rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2342 	if (!rsec)
2343 		return 0;
2344 
2345 	for_each_reloc(rsec, reloc) {
2346 		if (reloc->sym->type != STT_SECTION) {
2347 			WARN("unexpected relocation symbol type in %s", rsec->name);
2348 			return -1;
2349 		}
2350 
2351 		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2352 		if (!insn) {
2353 			WARN("bad .discard.retpoline_safe entry");
2354 			return -1;
2355 		}
2356 
2357 		if (insn->type != INSN_JUMP_DYNAMIC &&
2358 		    insn->type != INSN_CALL_DYNAMIC &&
2359 		    insn->type != INSN_RETURN &&
2360 		    insn->type != INSN_NOP) {
2361 			WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2362 			return -1;
2363 		}
2364 
2365 		insn->retpoline_safe = true;
2366 	}
2367 
2368 	return 0;
2369 }
2370 
2371 static int read_instr_hints(struct objtool_file *file)
2372 {
2373 	struct section *rsec;
2374 	struct instruction *insn;
2375 	struct reloc *reloc;
2376 
2377 	rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2378 	if (!rsec)
2379 		return 0;
2380 
2381 	for_each_reloc(rsec, reloc) {
2382 		if (reloc->sym->type != STT_SECTION) {
2383 			WARN("unexpected relocation symbol type in %s", rsec->name);
2384 			return -1;
2385 		}
2386 
2387 		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2388 		if (!insn) {
2389 			WARN("bad .discard.instr_end entry");
2390 			return -1;
2391 		}
2392 
2393 		insn->instr--;
2394 	}
2395 
2396 	rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2397 	if (!rsec)
2398 		return 0;
2399 
2400 	for_each_reloc(rsec, reloc) {
2401 		if (reloc->sym->type != STT_SECTION) {
2402 			WARN("unexpected relocation symbol type in %s", rsec->name);
2403 			return -1;
2404 		}
2405 
2406 		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2407 		if (!insn) {
2408 			WARN("bad .discard.instr_begin entry");
2409 			return -1;
2410 		}
2411 
2412 		insn->instr++;
2413 	}
2414 
2415 	return 0;
2416 }
2417 
2418 static int read_validate_unret_hints(struct objtool_file *file)
2419 {
2420 	struct section *rsec;
2421 	struct instruction *insn;
2422 	struct reloc *reloc;
2423 
2424 	rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2425 	if (!rsec)
2426 		return 0;
2427 
2428 	for_each_reloc(rsec, reloc) {
2429 		if (reloc->sym->type != STT_SECTION) {
2430 			WARN("unexpected relocation symbol type in %s", rsec->name);
2431 			return -1;
2432 		}
2433 
2434 		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2435 		if (!insn) {
2436 			WARN("bad .discard.instr_end entry");
2437 			return -1;
2438 		}
2439 		insn->unret = 1;
2440 	}
2441 
2442 	return 0;
2443 }
2444 
2445 
2446 static int read_intra_function_calls(struct objtool_file *file)
2447 {
2448 	struct instruction *insn;
2449 	struct section *rsec;
2450 	struct reloc *reloc;
2451 
2452 	rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2453 	if (!rsec)
2454 		return 0;
2455 
2456 	for_each_reloc(rsec, reloc) {
2457 		unsigned long dest_off;
2458 
2459 		if (reloc->sym->type != STT_SECTION) {
2460 			WARN("unexpected relocation symbol type in %s",
2461 			     rsec->name);
2462 			return -1;
2463 		}
2464 
2465 		insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2466 		if (!insn) {
2467 			WARN("bad .discard.intra_function_call entry");
2468 			return -1;
2469 		}
2470 
2471 		if (insn->type != INSN_CALL) {
2472 			WARN_INSN(insn, "intra_function_call not a direct call");
2473 			return -1;
2474 		}
2475 
2476 		/*
2477 		 * Treat intra-function CALLs as JMPs, but with a stack_op.
2478 		 * See add_call_destinations(), which strips stack_ops from
2479 		 * normal CALLs.
2480 		 */
2481 		insn->type = INSN_JUMP_UNCONDITIONAL;
2482 
2483 		dest_off = arch_jump_destination(insn);
2484 		insn->jump_dest = find_insn(file, insn->sec, dest_off);
2485 		if (!insn->jump_dest) {
2486 			WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2487 				  insn->sec->name, dest_off);
2488 			return -1;
2489 		}
2490 	}
2491 
2492 	return 0;
2493 }
2494 
2495 /*
2496  * Return true if name matches an instrumentation function, where calls to that
2497  * function from noinstr code can safely be removed, but compilers won't do so.
2498  */
2499 static bool is_profiling_func(const char *name)
2500 {
2501 	/*
2502 	 * Many compilers cannot disable KCOV with a function attribute.
2503 	 */
2504 	if (!strncmp(name, "__sanitizer_cov_", 16))
2505 		return true;
2506 
2507 	/*
2508 	 * Some compilers currently do not remove __tsan_func_entry/exit nor
2509 	 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2510 	 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2511 	 * minimum Clang version is 14.0, this can be removed.
2512 	 */
2513 	if (!strncmp(name, "__tsan_func_", 12) ||
2514 	    !strcmp(name, "__tsan_atomic_signal_fence"))
2515 		return true;
2516 
2517 	return false;
2518 }
2519 
2520 static int classify_symbols(struct objtool_file *file)
2521 {
2522 	struct symbol *func;
2523 
2524 	for_each_sym(file, func) {
2525 		if (func->bind != STB_GLOBAL)
2526 			continue;
2527 
2528 		if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2529 			     strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2530 			func->static_call_tramp = true;
2531 
2532 		if (arch_is_retpoline(func))
2533 			func->retpoline_thunk = true;
2534 
2535 		if (arch_is_rethunk(func))
2536 			func->return_thunk = true;
2537 
2538 		if (arch_is_embedded_insn(func))
2539 			func->embedded_insn = true;
2540 
2541 		if (arch_ftrace_match(func->name))
2542 			func->fentry = true;
2543 
2544 		if (is_profiling_func(func->name))
2545 			func->profiling_func = true;
2546 	}
2547 
2548 	return 0;
2549 }
2550 
2551 static void mark_rodata(struct objtool_file *file)
2552 {
2553 	struct section *sec;
2554 	bool found = false;
2555 
2556 	/*
2557 	 * Search for the following rodata sections, each of which can
2558 	 * potentially contain jump tables:
2559 	 *
2560 	 * - .rodata: can contain GCC switch tables
2561 	 * - .rodata.<func>: same, if -fdata-sections is being used
2562 	 * - .rodata..c_jump_table: contains C annotated jump tables
2563 	 *
2564 	 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2565 	 */
2566 	for_each_sec(file, sec) {
2567 		if (!strncmp(sec->name, ".rodata", 7) &&
2568 		    !strstr(sec->name, ".str1.")) {
2569 			sec->rodata = true;
2570 			found = true;
2571 		}
2572 	}
2573 
2574 	file->rodata = found;
2575 }
2576 
2577 static int decode_sections(struct objtool_file *file)
2578 {
2579 	int ret;
2580 
2581 	mark_rodata(file);
2582 
2583 	ret = init_pv_ops(file);
2584 	if (ret)
2585 		return ret;
2586 
2587 	/*
2588 	 * Must be before add_{jump_call}_destination.
2589 	 */
2590 	ret = classify_symbols(file);
2591 	if (ret)
2592 		return ret;
2593 
2594 	ret = decode_instructions(file);
2595 	if (ret)
2596 		return ret;
2597 
2598 	add_ignores(file);
2599 	add_uaccess_safe(file);
2600 
2601 	ret = add_ignore_alternatives(file);
2602 	if (ret)
2603 		return ret;
2604 
2605 	/*
2606 	 * Must be before read_unwind_hints() since that needs insn->noendbr.
2607 	 */
2608 	ret = read_noendbr_hints(file);
2609 	if (ret)
2610 		return ret;
2611 
2612 	/*
2613 	 * Must be before add_jump_destinations(), which depends on 'func'
2614 	 * being set for alternatives, to enable proper sibling call detection.
2615 	 */
2616 	if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2617 		ret = add_special_section_alts(file);
2618 		if (ret)
2619 			return ret;
2620 	}
2621 
2622 	ret = add_jump_destinations(file);
2623 	if (ret)
2624 		return ret;
2625 
2626 	/*
2627 	 * Must be before add_call_destination(); it changes INSN_CALL to
2628 	 * INSN_JUMP.
2629 	 */
2630 	ret = read_intra_function_calls(file);
2631 	if (ret)
2632 		return ret;
2633 
2634 	ret = add_call_destinations(file);
2635 	if (ret)
2636 		return ret;
2637 
2638 	/*
2639 	 * Must be after add_call_destinations() such that it can override
2640 	 * dead_end_function() marks.
2641 	 */
2642 	ret = add_dead_ends(file);
2643 	if (ret)
2644 		return ret;
2645 
2646 	ret = add_jump_table_alts(file);
2647 	if (ret)
2648 		return ret;
2649 
2650 	ret = read_unwind_hints(file);
2651 	if (ret)
2652 		return ret;
2653 
2654 	ret = read_retpoline_hints(file);
2655 	if (ret)
2656 		return ret;
2657 
2658 	ret = read_instr_hints(file);
2659 	if (ret)
2660 		return ret;
2661 
2662 	ret = read_validate_unret_hints(file);
2663 	if (ret)
2664 		return ret;
2665 
2666 	return 0;
2667 }
2668 
2669 static bool is_special_call(struct instruction *insn)
2670 {
2671 	if (insn->type == INSN_CALL) {
2672 		struct symbol *dest = insn_call_dest(insn);
2673 
2674 		if (!dest)
2675 			return false;
2676 
2677 		if (dest->fentry || dest->embedded_insn)
2678 			return true;
2679 	}
2680 
2681 	return false;
2682 }
2683 
2684 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2685 {
2686 	struct cfi_state *cfi = &state->cfi;
2687 	int i;
2688 
2689 	if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2690 		return true;
2691 
2692 	if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2693 		return true;
2694 
2695 	if (cfi->stack_size != initial_func_cfi.cfa.offset)
2696 		return true;
2697 
2698 	for (i = 0; i < CFI_NUM_REGS; i++) {
2699 		if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2700 		    cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2701 			return true;
2702 	}
2703 
2704 	return false;
2705 }
2706 
2707 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2708 				int expected_offset)
2709 {
2710 	return reg->base == CFI_CFA &&
2711 	       reg->offset == expected_offset;
2712 }
2713 
2714 static bool has_valid_stack_frame(struct insn_state *state)
2715 {
2716 	struct cfi_state *cfi = &state->cfi;
2717 
2718 	if (cfi->cfa.base == CFI_BP &&
2719 	    check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2720 	    check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2721 		return true;
2722 
2723 	if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2724 		return true;
2725 
2726 	return false;
2727 }
2728 
2729 static int update_cfi_state_regs(struct instruction *insn,
2730 				  struct cfi_state *cfi,
2731 				  struct stack_op *op)
2732 {
2733 	struct cfi_reg *cfa = &cfi->cfa;
2734 
2735 	if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2736 		return 0;
2737 
2738 	/* push */
2739 	if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2740 		cfa->offset += 8;
2741 
2742 	/* pop */
2743 	if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2744 		cfa->offset -= 8;
2745 
2746 	/* add immediate to sp */
2747 	if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2748 	    op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2749 		cfa->offset -= op->src.offset;
2750 
2751 	return 0;
2752 }
2753 
2754 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2755 {
2756 	if (arch_callee_saved_reg(reg) &&
2757 	    cfi->regs[reg].base == CFI_UNDEFINED) {
2758 		cfi->regs[reg].base = base;
2759 		cfi->regs[reg].offset = offset;
2760 	}
2761 }
2762 
2763 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2764 {
2765 	cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2766 	cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2767 }
2768 
2769 /*
2770  * A note about DRAP stack alignment:
2771  *
2772  * GCC has the concept of a DRAP register, which is used to help keep track of
2773  * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2774  * register.  The typical DRAP pattern is:
2775  *
2776  *   4c 8d 54 24 08		lea    0x8(%rsp),%r10
2777  *   48 83 e4 c0		and    $0xffffffffffffffc0,%rsp
2778  *   41 ff 72 f8		pushq  -0x8(%r10)
2779  *   55				push   %rbp
2780  *   48 89 e5			mov    %rsp,%rbp
2781  *				(more pushes)
2782  *   41 52			push   %r10
2783  *				...
2784  *   41 5a			pop    %r10
2785  *				(more pops)
2786  *   5d				pop    %rbp
2787  *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2788  *   c3				retq
2789  *
2790  * There are some variations in the epilogues, like:
2791  *
2792  *   5b				pop    %rbx
2793  *   41 5a			pop    %r10
2794  *   41 5c			pop    %r12
2795  *   41 5d			pop    %r13
2796  *   41 5e			pop    %r14
2797  *   c9				leaveq
2798  *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2799  *   c3				retq
2800  *
2801  * and:
2802  *
2803  *   4c 8b 55 e8		mov    -0x18(%rbp),%r10
2804  *   48 8b 5d e0		mov    -0x20(%rbp),%rbx
2805  *   4c 8b 65 f0		mov    -0x10(%rbp),%r12
2806  *   4c 8b 6d f8		mov    -0x8(%rbp),%r13
2807  *   c9				leaveq
2808  *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2809  *   c3				retq
2810  *
2811  * Sometimes r13 is used as the DRAP register, in which case it's saved and
2812  * restored beforehand:
2813  *
2814  *   41 55			push   %r13
2815  *   4c 8d 6c 24 10		lea    0x10(%rsp),%r13
2816  *   48 83 e4 f0		and    $0xfffffffffffffff0,%rsp
2817  *				...
2818  *   49 8d 65 f0		lea    -0x10(%r13),%rsp
2819  *   41 5d			pop    %r13
2820  *   c3				retq
2821  */
2822 static int update_cfi_state(struct instruction *insn,
2823 			    struct instruction *next_insn,
2824 			    struct cfi_state *cfi, struct stack_op *op)
2825 {
2826 	struct cfi_reg *cfa = &cfi->cfa;
2827 	struct cfi_reg *regs = cfi->regs;
2828 
2829 	/* ignore UNWIND_HINT_UNDEFINED regions */
2830 	if (cfi->force_undefined)
2831 		return 0;
2832 
2833 	/* stack operations don't make sense with an undefined CFA */
2834 	if (cfa->base == CFI_UNDEFINED) {
2835 		if (insn_func(insn)) {
2836 			WARN_INSN(insn, "undefined stack state");
2837 			return -1;
2838 		}
2839 		return 0;
2840 	}
2841 
2842 	if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2843 	    cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2844 		return update_cfi_state_regs(insn, cfi, op);
2845 
2846 	switch (op->dest.type) {
2847 
2848 	case OP_DEST_REG:
2849 		switch (op->src.type) {
2850 
2851 		case OP_SRC_REG:
2852 			if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2853 			    cfa->base == CFI_SP &&
2854 			    check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2855 
2856 				/* mov %rsp, %rbp */
2857 				cfa->base = op->dest.reg;
2858 				cfi->bp_scratch = false;
2859 			}
2860 
2861 			else if (op->src.reg == CFI_SP &&
2862 				 op->dest.reg == CFI_BP && cfi->drap) {
2863 
2864 				/* drap: mov %rsp, %rbp */
2865 				regs[CFI_BP].base = CFI_BP;
2866 				regs[CFI_BP].offset = -cfi->stack_size;
2867 				cfi->bp_scratch = false;
2868 			}
2869 
2870 			else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2871 
2872 				/*
2873 				 * mov %rsp, %reg
2874 				 *
2875 				 * This is needed for the rare case where GCC
2876 				 * does:
2877 				 *
2878 				 *   mov    %rsp, %rax
2879 				 *   ...
2880 				 *   mov    %rax, %rsp
2881 				 */
2882 				cfi->vals[op->dest.reg].base = CFI_CFA;
2883 				cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2884 			}
2885 
2886 			else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2887 				 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2888 
2889 				/*
2890 				 * mov %rbp, %rsp
2891 				 *
2892 				 * Restore the original stack pointer (Clang).
2893 				 */
2894 				cfi->stack_size = -cfi->regs[CFI_BP].offset;
2895 			}
2896 
2897 			else if (op->dest.reg == cfa->base) {
2898 
2899 				/* mov %reg, %rsp */
2900 				if (cfa->base == CFI_SP &&
2901 				    cfi->vals[op->src.reg].base == CFI_CFA) {
2902 
2903 					/*
2904 					 * This is needed for the rare case
2905 					 * where GCC does something dumb like:
2906 					 *
2907 					 *   lea    0x8(%rsp), %rcx
2908 					 *   ...
2909 					 *   mov    %rcx, %rsp
2910 					 */
2911 					cfa->offset = -cfi->vals[op->src.reg].offset;
2912 					cfi->stack_size = cfa->offset;
2913 
2914 				} else if (cfa->base == CFI_SP &&
2915 					   cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2916 					   cfi->vals[op->src.reg].offset == cfa->offset) {
2917 
2918 					/*
2919 					 * Stack swizzle:
2920 					 *
2921 					 * 1: mov %rsp, (%[tos])
2922 					 * 2: mov %[tos], %rsp
2923 					 *    ...
2924 					 * 3: pop %rsp
2925 					 *
2926 					 * Where:
2927 					 *
2928 					 * 1 - places a pointer to the previous
2929 					 *     stack at the Top-of-Stack of the
2930 					 *     new stack.
2931 					 *
2932 					 * 2 - switches to the new stack.
2933 					 *
2934 					 * 3 - pops the Top-of-Stack to restore
2935 					 *     the original stack.
2936 					 *
2937 					 * Note: we set base to SP_INDIRECT
2938 					 * here and preserve offset. Therefore
2939 					 * when the unwinder reaches ToS it
2940 					 * will dereference SP and then add the
2941 					 * offset to find the next frame, IOW:
2942 					 * (%rsp) + offset.
2943 					 */
2944 					cfa->base = CFI_SP_INDIRECT;
2945 
2946 				} else {
2947 					cfa->base = CFI_UNDEFINED;
2948 					cfa->offset = 0;
2949 				}
2950 			}
2951 
2952 			else if (op->dest.reg == CFI_SP &&
2953 				 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2954 				 cfi->vals[op->src.reg].offset == cfa->offset) {
2955 
2956 				/*
2957 				 * The same stack swizzle case 2) as above. But
2958 				 * because we can't change cfa->base, case 3)
2959 				 * will become a regular POP. Pretend we're a
2960 				 * PUSH so things don't go unbalanced.
2961 				 */
2962 				cfi->stack_size += 8;
2963 			}
2964 
2965 
2966 			break;
2967 
2968 		case OP_SRC_ADD:
2969 			if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2970 
2971 				/* add imm, %rsp */
2972 				cfi->stack_size -= op->src.offset;
2973 				if (cfa->base == CFI_SP)
2974 					cfa->offset -= op->src.offset;
2975 				break;
2976 			}
2977 
2978 			if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2979 
2980 				/* lea disp(%rbp), %rsp */
2981 				cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2982 				break;
2983 			}
2984 
2985 			if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2986 
2987 				/* drap: lea disp(%rsp), %drap */
2988 				cfi->drap_reg = op->dest.reg;
2989 
2990 				/*
2991 				 * lea disp(%rsp), %reg
2992 				 *
2993 				 * This is needed for the rare case where GCC
2994 				 * does something dumb like:
2995 				 *
2996 				 *   lea    0x8(%rsp), %rcx
2997 				 *   ...
2998 				 *   mov    %rcx, %rsp
2999 				 */
3000 				cfi->vals[op->dest.reg].base = CFI_CFA;
3001 				cfi->vals[op->dest.reg].offset = \
3002 					-cfi->stack_size + op->src.offset;
3003 
3004 				break;
3005 			}
3006 
3007 			if (cfi->drap && op->dest.reg == CFI_SP &&
3008 			    op->src.reg == cfi->drap_reg) {
3009 
3010 				 /* drap: lea disp(%drap), %rsp */
3011 				cfa->base = CFI_SP;
3012 				cfa->offset = cfi->stack_size = -op->src.offset;
3013 				cfi->drap_reg = CFI_UNDEFINED;
3014 				cfi->drap = false;
3015 				break;
3016 			}
3017 
3018 			if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3019 				WARN_INSN(insn, "unsupported stack register modification");
3020 				return -1;
3021 			}
3022 
3023 			break;
3024 
3025 		case OP_SRC_AND:
3026 			if (op->dest.reg != CFI_SP ||
3027 			    (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3028 			    (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3029 				WARN_INSN(insn, "unsupported stack pointer realignment");
3030 				return -1;
3031 			}
3032 
3033 			if (cfi->drap_reg != CFI_UNDEFINED) {
3034 				/* drap: and imm, %rsp */
3035 				cfa->base = cfi->drap_reg;
3036 				cfa->offset = cfi->stack_size = 0;
3037 				cfi->drap = true;
3038 			}
3039 
3040 			/*
3041 			 * Older versions of GCC (4.8ish) realign the stack
3042 			 * without DRAP, with a frame pointer.
3043 			 */
3044 
3045 			break;
3046 
3047 		case OP_SRC_POP:
3048 		case OP_SRC_POPF:
3049 			if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3050 
3051 				/* pop %rsp; # restore from a stack swizzle */
3052 				cfa->base = CFI_SP;
3053 				break;
3054 			}
3055 
3056 			if (!cfi->drap && op->dest.reg == cfa->base) {
3057 
3058 				/* pop %rbp */
3059 				cfa->base = CFI_SP;
3060 			}
3061 
3062 			if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3063 			    op->dest.reg == cfi->drap_reg &&
3064 			    cfi->drap_offset == -cfi->stack_size) {
3065 
3066 				/* drap: pop %drap */
3067 				cfa->base = cfi->drap_reg;
3068 				cfa->offset = 0;
3069 				cfi->drap_offset = -1;
3070 
3071 			} else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3072 
3073 				/* pop %reg */
3074 				restore_reg(cfi, op->dest.reg);
3075 			}
3076 
3077 			cfi->stack_size -= 8;
3078 			if (cfa->base == CFI_SP)
3079 				cfa->offset -= 8;
3080 
3081 			break;
3082 
3083 		case OP_SRC_REG_INDIRECT:
3084 			if (!cfi->drap && op->dest.reg == cfa->base &&
3085 			    op->dest.reg == CFI_BP) {
3086 
3087 				/* mov disp(%rsp), %rbp */
3088 				cfa->base = CFI_SP;
3089 				cfa->offset = cfi->stack_size;
3090 			}
3091 
3092 			if (cfi->drap && op->src.reg == CFI_BP &&
3093 			    op->src.offset == cfi->drap_offset) {
3094 
3095 				/* drap: mov disp(%rbp), %drap */
3096 				cfa->base = cfi->drap_reg;
3097 				cfa->offset = 0;
3098 				cfi->drap_offset = -1;
3099 			}
3100 
3101 			if (cfi->drap && op->src.reg == CFI_BP &&
3102 			    op->src.offset == regs[op->dest.reg].offset) {
3103 
3104 				/* drap: mov disp(%rbp), %reg */
3105 				restore_reg(cfi, op->dest.reg);
3106 
3107 			} else if (op->src.reg == cfa->base &&
3108 			    op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3109 
3110 				/* mov disp(%rbp), %reg */
3111 				/* mov disp(%rsp), %reg */
3112 				restore_reg(cfi, op->dest.reg);
3113 
3114 			} else if (op->src.reg == CFI_SP &&
3115 				   op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3116 
3117 				/* mov disp(%rsp), %reg */
3118 				restore_reg(cfi, op->dest.reg);
3119 			}
3120 
3121 			break;
3122 
3123 		default:
3124 			WARN_INSN(insn, "unknown stack-related instruction");
3125 			return -1;
3126 		}
3127 
3128 		break;
3129 
3130 	case OP_DEST_PUSH:
3131 	case OP_DEST_PUSHF:
3132 		cfi->stack_size += 8;
3133 		if (cfa->base == CFI_SP)
3134 			cfa->offset += 8;
3135 
3136 		if (op->src.type != OP_SRC_REG)
3137 			break;
3138 
3139 		if (cfi->drap) {
3140 			if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3141 
3142 				/* drap: push %drap */
3143 				cfa->base = CFI_BP_INDIRECT;
3144 				cfa->offset = -cfi->stack_size;
3145 
3146 				/* save drap so we know when to restore it */
3147 				cfi->drap_offset = -cfi->stack_size;
3148 
3149 			} else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3150 
3151 				/* drap: push %rbp */
3152 				cfi->stack_size = 0;
3153 
3154 			} else {
3155 
3156 				/* drap: push %reg */
3157 				save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3158 			}
3159 
3160 		} else {
3161 
3162 			/* push %reg */
3163 			save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3164 		}
3165 
3166 		/* detect when asm code uses rbp as a scratch register */
3167 		if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3168 		    cfa->base != CFI_BP)
3169 			cfi->bp_scratch = true;
3170 		break;
3171 
3172 	case OP_DEST_REG_INDIRECT:
3173 
3174 		if (cfi->drap) {
3175 			if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3176 
3177 				/* drap: mov %drap, disp(%rbp) */
3178 				cfa->base = CFI_BP_INDIRECT;
3179 				cfa->offset = op->dest.offset;
3180 
3181 				/* save drap offset so we know when to restore it */
3182 				cfi->drap_offset = op->dest.offset;
3183 			} else {
3184 
3185 				/* drap: mov reg, disp(%rbp) */
3186 				save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3187 			}
3188 
3189 		} else if (op->dest.reg == cfa->base) {
3190 
3191 			/* mov reg, disp(%rbp) */
3192 			/* mov reg, disp(%rsp) */
3193 			save_reg(cfi, op->src.reg, CFI_CFA,
3194 				 op->dest.offset - cfi->cfa.offset);
3195 
3196 		} else if (op->dest.reg == CFI_SP) {
3197 
3198 			/* mov reg, disp(%rsp) */
3199 			save_reg(cfi, op->src.reg, CFI_CFA,
3200 				 op->dest.offset - cfi->stack_size);
3201 
3202 		} else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3203 
3204 			/* mov %rsp, (%reg); # setup a stack swizzle. */
3205 			cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3206 			cfi->vals[op->dest.reg].offset = cfa->offset;
3207 		}
3208 
3209 		break;
3210 
3211 	case OP_DEST_MEM:
3212 		if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3213 			WARN_INSN(insn, "unknown stack-related memory operation");
3214 			return -1;
3215 		}
3216 
3217 		/* pop mem */
3218 		cfi->stack_size -= 8;
3219 		if (cfa->base == CFI_SP)
3220 			cfa->offset -= 8;
3221 
3222 		break;
3223 
3224 	default:
3225 		WARN_INSN(insn, "unknown stack-related instruction");
3226 		return -1;
3227 	}
3228 
3229 	return 0;
3230 }
3231 
3232 /*
3233  * The stack layouts of alternatives instructions can sometimes diverge when
3234  * they have stack modifications.  That's fine as long as the potential stack
3235  * layouts don't conflict at any given potential instruction boundary.
3236  *
3237  * Flatten the CFIs of the different alternative code streams (both original
3238  * and replacement) into a single shared CFI array which can be used to detect
3239  * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3240  */
3241 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3242 {
3243 	struct cfi_state **alt_cfi;
3244 	int group_off;
3245 
3246 	if (!insn->alt_group)
3247 		return 0;
3248 
3249 	if (!insn->cfi) {
3250 		WARN("CFI missing");
3251 		return -1;
3252 	}
3253 
3254 	alt_cfi = insn->alt_group->cfi;
3255 	group_off = insn->offset - insn->alt_group->first_insn->offset;
3256 
3257 	if (!alt_cfi[group_off]) {
3258 		alt_cfi[group_off] = insn->cfi;
3259 	} else {
3260 		if (cficmp(alt_cfi[group_off], insn->cfi)) {
3261 			struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3262 			struct instruction *orig = orig_group->first_insn;
3263 			char *where = offstr(insn->sec, insn->offset);
3264 			WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3265 			free(where);
3266 			return -1;
3267 		}
3268 	}
3269 
3270 	return 0;
3271 }
3272 
3273 static int handle_insn_ops(struct instruction *insn,
3274 			   struct instruction *next_insn,
3275 			   struct insn_state *state)
3276 {
3277 	struct stack_op *op;
3278 
3279 	for (op = insn->stack_ops; op; op = op->next) {
3280 
3281 		if (update_cfi_state(insn, next_insn, &state->cfi, op))
3282 			return 1;
3283 
3284 		if (!insn->alt_group)
3285 			continue;
3286 
3287 		if (op->dest.type == OP_DEST_PUSHF) {
3288 			if (!state->uaccess_stack) {
3289 				state->uaccess_stack = 1;
3290 			} else if (state->uaccess_stack >> 31) {
3291 				WARN_INSN(insn, "PUSHF stack exhausted");
3292 				return 1;
3293 			}
3294 			state->uaccess_stack <<= 1;
3295 			state->uaccess_stack  |= state->uaccess;
3296 		}
3297 
3298 		if (op->src.type == OP_SRC_POPF) {
3299 			if (state->uaccess_stack) {
3300 				state->uaccess = state->uaccess_stack & 1;
3301 				state->uaccess_stack >>= 1;
3302 				if (state->uaccess_stack == 1)
3303 					state->uaccess_stack = 0;
3304 			}
3305 		}
3306 	}
3307 
3308 	return 0;
3309 }
3310 
3311 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3312 {
3313 	struct cfi_state *cfi1 = insn->cfi;
3314 	int i;
3315 
3316 	if (!cfi1) {
3317 		WARN("CFI missing");
3318 		return false;
3319 	}
3320 
3321 	if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3322 
3323 		WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3324 			  cfi1->cfa.base, cfi1->cfa.offset,
3325 			  cfi2->cfa.base, cfi2->cfa.offset);
3326 
3327 	} else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3328 		for (i = 0; i < CFI_NUM_REGS; i++) {
3329 			if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3330 				    sizeof(struct cfi_reg)))
3331 				continue;
3332 
3333 			WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3334 				  i, cfi1->regs[i].base, cfi1->regs[i].offset,
3335 				  i, cfi2->regs[i].base, cfi2->regs[i].offset);
3336 			break;
3337 		}
3338 
3339 	} else if (cfi1->type != cfi2->type) {
3340 
3341 		WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3342 			  cfi1->type, cfi2->type);
3343 
3344 	} else if (cfi1->drap != cfi2->drap ||
3345 		   (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3346 		   (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3347 
3348 		WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3349 			  cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3350 			  cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3351 
3352 	} else
3353 		return true;
3354 
3355 	return false;
3356 }
3357 
3358 static inline bool func_uaccess_safe(struct symbol *func)
3359 {
3360 	if (func)
3361 		return func->uaccess_safe;
3362 
3363 	return false;
3364 }
3365 
3366 static inline const char *call_dest_name(struct instruction *insn)
3367 {
3368 	static char pvname[19];
3369 	struct reloc *reloc;
3370 	int idx;
3371 
3372 	if (insn_call_dest(insn))
3373 		return insn_call_dest(insn)->name;
3374 
3375 	reloc = insn_reloc(NULL, insn);
3376 	if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3377 		idx = (reloc_addend(reloc) / sizeof(void *));
3378 		snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3379 		return pvname;
3380 	}
3381 
3382 	return "{dynamic}";
3383 }
3384 
3385 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3386 {
3387 	struct symbol *target;
3388 	struct reloc *reloc;
3389 	int idx;
3390 
3391 	reloc = insn_reloc(file, insn);
3392 	if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3393 		return false;
3394 
3395 	idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3396 
3397 	if (file->pv_ops[idx].clean)
3398 		return true;
3399 
3400 	file->pv_ops[idx].clean = true;
3401 
3402 	list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3403 		if (!target->sec->noinstr) {
3404 			WARN("pv_ops[%d]: %s", idx, target->name);
3405 			file->pv_ops[idx].clean = false;
3406 		}
3407 	}
3408 
3409 	return file->pv_ops[idx].clean;
3410 }
3411 
3412 static inline bool noinstr_call_dest(struct objtool_file *file,
3413 				     struct instruction *insn,
3414 				     struct symbol *func)
3415 {
3416 	/*
3417 	 * We can't deal with indirect function calls at present;
3418 	 * assume they're instrumented.
3419 	 */
3420 	if (!func) {
3421 		if (file->pv_ops)
3422 			return pv_call_dest(file, insn);
3423 
3424 		return false;
3425 	}
3426 
3427 	/*
3428 	 * If the symbol is from a noinstr section; we good.
3429 	 */
3430 	if (func->sec->noinstr)
3431 		return true;
3432 
3433 	/*
3434 	 * If the symbol is a static_call trampoline, we can't tell.
3435 	 */
3436 	if (func->static_call_tramp)
3437 		return true;
3438 
3439 	/*
3440 	 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3441 	 * something 'BAD' happened. At the risk of taking the machine down,
3442 	 * let them proceed to get the message out.
3443 	 */
3444 	if (!strncmp(func->name, "__ubsan_handle_", 15))
3445 		return true;
3446 
3447 	return false;
3448 }
3449 
3450 static int validate_call(struct objtool_file *file,
3451 			 struct instruction *insn,
3452 			 struct insn_state *state)
3453 {
3454 	if (state->noinstr && state->instr <= 0 &&
3455 	    !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3456 		WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3457 		return 1;
3458 	}
3459 
3460 	if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3461 		WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3462 		return 1;
3463 	}
3464 
3465 	if (state->df) {
3466 		WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3467 		return 1;
3468 	}
3469 
3470 	return 0;
3471 }
3472 
3473 static int validate_sibling_call(struct objtool_file *file,
3474 				 struct instruction *insn,
3475 				 struct insn_state *state)
3476 {
3477 	if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3478 		WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3479 		return 1;
3480 	}
3481 
3482 	return validate_call(file, insn, state);
3483 }
3484 
3485 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3486 {
3487 	if (state->noinstr && state->instr > 0) {
3488 		WARN_INSN(insn, "return with instrumentation enabled");
3489 		return 1;
3490 	}
3491 
3492 	if (state->uaccess && !func_uaccess_safe(func)) {
3493 		WARN_INSN(insn, "return with UACCESS enabled");
3494 		return 1;
3495 	}
3496 
3497 	if (!state->uaccess && func_uaccess_safe(func)) {
3498 		WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3499 		return 1;
3500 	}
3501 
3502 	if (state->df) {
3503 		WARN_INSN(insn, "return with DF set");
3504 		return 1;
3505 	}
3506 
3507 	if (func && has_modified_stack_frame(insn, state)) {
3508 		WARN_INSN(insn, "return with modified stack frame");
3509 		return 1;
3510 	}
3511 
3512 	if (state->cfi.bp_scratch) {
3513 		WARN_INSN(insn, "BP used as a scratch register");
3514 		return 1;
3515 	}
3516 
3517 	return 0;
3518 }
3519 
3520 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3521 						 struct instruction *insn)
3522 {
3523 	struct alt_group *alt_group = insn->alt_group;
3524 
3525 	/*
3526 	 * Simulate the fact that alternatives are patched in-place.  When the
3527 	 * end of a replacement alt_group is reached, redirect objtool flow to
3528 	 * the end of the original alt_group.
3529 	 *
3530 	 * insn->alts->insn -> alt_group->first_insn
3531 	 *		       ...
3532 	 *		       alt_group->last_insn
3533 	 *		       [alt_group->nop]      -> next(orig_group->last_insn)
3534 	 */
3535 	if (alt_group) {
3536 		if (alt_group->nop) {
3537 			/* ->nop implies ->orig_group */
3538 			if (insn == alt_group->last_insn)
3539 				return alt_group->nop;
3540 			if (insn == alt_group->nop)
3541 				goto next_orig;
3542 		}
3543 		if (insn == alt_group->last_insn && alt_group->orig_group)
3544 			goto next_orig;
3545 	}
3546 
3547 	return next_insn_same_sec(file, insn);
3548 
3549 next_orig:
3550 	return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3551 }
3552 
3553 /*
3554  * Follow the branch starting at the given instruction, and recursively follow
3555  * any other branches (jumps).  Meanwhile, track the frame pointer state at
3556  * each instruction and validate all the rules described in
3557  * tools/objtool/Documentation/objtool.txt.
3558  */
3559 static int validate_branch(struct objtool_file *file, struct symbol *func,
3560 			   struct instruction *insn, struct insn_state state)
3561 {
3562 	struct alternative *alt;
3563 	struct instruction *next_insn, *prev_insn = NULL;
3564 	struct section *sec;
3565 	u8 visited;
3566 	int ret;
3567 
3568 	sec = insn->sec;
3569 
3570 	while (1) {
3571 		next_insn = next_insn_to_validate(file, insn);
3572 
3573 		if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3574 			/* Ignore KCFI type preambles, which always fall through */
3575 			if (!strncmp(func->name, "__cfi_", 6) ||
3576 			    !strncmp(func->name, "__pfx_", 6))
3577 				return 0;
3578 
3579 			WARN("%s() falls through to next function %s()",
3580 			     func->name, insn_func(insn)->name);
3581 			return 1;
3582 		}
3583 
3584 		if (func && insn->ignore) {
3585 			WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3586 			return 1;
3587 		}
3588 
3589 		visited = VISITED_BRANCH << state.uaccess;
3590 		if (insn->visited & VISITED_BRANCH_MASK) {
3591 			if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3592 				return 1;
3593 
3594 			if (insn->visited & visited)
3595 				return 0;
3596 		} else {
3597 			nr_insns_visited++;
3598 		}
3599 
3600 		if (state.noinstr)
3601 			state.instr += insn->instr;
3602 
3603 		if (insn->hint) {
3604 			if (insn->restore) {
3605 				struct instruction *save_insn, *i;
3606 
3607 				i = insn;
3608 				save_insn = NULL;
3609 
3610 				sym_for_each_insn_continue_reverse(file, func, i) {
3611 					if (i->save) {
3612 						save_insn = i;
3613 						break;
3614 					}
3615 				}
3616 
3617 				if (!save_insn) {
3618 					WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3619 					return 1;
3620 				}
3621 
3622 				if (!save_insn->visited) {
3623 					/*
3624 					 * If the restore hint insn is at the
3625 					 * beginning of a basic block and was
3626 					 * branched to from elsewhere, and the
3627 					 * save insn hasn't been visited yet,
3628 					 * defer following this branch for now.
3629 					 * It will be seen later via the
3630 					 * straight-line path.
3631 					 */
3632 					if (!prev_insn)
3633 						return 0;
3634 
3635 					WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3636 					return 1;
3637 				}
3638 
3639 				insn->cfi = save_insn->cfi;
3640 				nr_cfi_reused++;
3641 			}
3642 
3643 			state.cfi = *insn->cfi;
3644 		} else {
3645 			/* XXX track if we actually changed state.cfi */
3646 
3647 			if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3648 				insn->cfi = prev_insn->cfi;
3649 				nr_cfi_reused++;
3650 			} else {
3651 				insn->cfi = cfi_hash_find_or_add(&state.cfi);
3652 			}
3653 		}
3654 
3655 		insn->visited |= visited;
3656 
3657 		if (propagate_alt_cfi(file, insn))
3658 			return 1;
3659 
3660 		if (!insn->ignore_alts && insn->alts) {
3661 			bool skip_orig = false;
3662 
3663 			for (alt = insn->alts; alt; alt = alt->next) {
3664 				if (alt->skip_orig)
3665 					skip_orig = true;
3666 
3667 				ret = validate_branch(file, func, alt->insn, state);
3668 				if (ret) {
3669 					BT_INSN(insn, "(alt)");
3670 					return ret;
3671 				}
3672 			}
3673 
3674 			if (skip_orig)
3675 				return 0;
3676 		}
3677 
3678 		if (handle_insn_ops(insn, next_insn, &state))
3679 			return 1;
3680 
3681 		switch (insn->type) {
3682 
3683 		case INSN_RETURN:
3684 			return validate_return(func, insn, &state);
3685 
3686 		case INSN_CALL:
3687 		case INSN_CALL_DYNAMIC:
3688 			ret = validate_call(file, insn, &state);
3689 			if (ret)
3690 				return ret;
3691 
3692 			if (opts.stackval && func && !is_special_call(insn) &&
3693 			    !has_valid_stack_frame(&state)) {
3694 				WARN_INSN(insn, "call without frame pointer save/setup");
3695 				return 1;
3696 			}
3697 
3698 			if (insn->dead_end)
3699 				return 0;
3700 
3701 			break;
3702 
3703 		case INSN_JUMP_CONDITIONAL:
3704 		case INSN_JUMP_UNCONDITIONAL:
3705 			if (is_sibling_call(insn)) {
3706 				ret = validate_sibling_call(file, insn, &state);
3707 				if (ret)
3708 					return ret;
3709 
3710 			} else if (insn->jump_dest) {
3711 				ret = validate_branch(file, func,
3712 						      insn->jump_dest, state);
3713 				if (ret) {
3714 					BT_INSN(insn, "(branch)");
3715 					return ret;
3716 				}
3717 			}
3718 
3719 			if (insn->type == INSN_JUMP_UNCONDITIONAL)
3720 				return 0;
3721 
3722 			break;
3723 
3724 		case INSN_JUMP_DYNAMIC:
3725 		case INSN_JUMP_DYNAMIC_CONDITIONAL:
3726 			if (is_sibling_call(insn)) {
3727 				ret = validate_sibling_call(file, insn, &state);
3728 				if (ret)
3729 					return ret;
3730 			}
3731 
3732 			if (insn->type == INSN_JUMP_DYNAMIC)
3733 				return 0;
3734 
3735 			break;
3736 
3737 		case INSN_CONTEXT_SWITCH:
3738 			if (func && (!next_insn || !next_insn->hint)) {
3739 				WARN_INSN(insn, "unsupported instruction in callable function");
3740 				return 1;
3741 			}
3742 			return 0;
3743 
3744 		case INSN_STAC:
3745 			if (state.uaccess) {
3746 				WARN_INSN(insn, "recursive UACCESS enable");
3747 				return 1;
3748 			}
3749 
3750 			state.uaccess = true;
3751 			break;
3752 
3753 		case INSN_CLAC:
3754 			if (!state.uaccess && func) {
3755 				WARN_INSN(insn, "redundant UACCESS disable");
3756 				return 1;
3757 			}
3758 
3759 			if (func_uaccess_safe(func) && !state.uaccess_stack) {
3760 				WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3761 				return 1;
3762 			}
3763 
3764 			state.uaccess = false;
3765 			break;
3766 
3767 		case INSN_STD:
3768 			if (state.df) {
3769 				WARN_INSN(insn, "recursive STD");
3770 				return 1;
3771 			}
3772 
3773 			state.df = true;
3774 			break;
3775 
3776 		case INSN_CLD:
3777 			if (!state.df && func) {
3778 				WARN_INSN(insn, "redundant CLD");
3779 				return 1;
3780 			}
3781 
3782 			state.df = false;
3783 			break;
3784 
3785 		default:
3786 			break;
3787 		}
3788 
3789 		if (insn->dead_end)
3790 			return 0;
3791 
3792 		if (!next_insn) {
3793 			if (state.cfi.cfa.base == CFI_UNDEFINED)
3794 				return 0;
3795 			WARN("%s: unexpected end of section", sec->name);
3796 			return 1;
3797 		}
3798 
3799 		prev_insn = insn;
3800 		insn = next_insn;
3801 	}
3802 
3803 	return 0;
3804 }
3805 
3806 static int validate_unwind_hint(struct objtool_file *file,
3807 				  struct instruction *insn,
3808 				  struct insn_state *state)
3809 {
3810 	if (insn->hint && !insn->visited && !insn->ignore) {
3811 		int ret = validate_branch(file, insn_func(insn), insn, *state);
3812 		if (ret)
3813 			BT_INSN(insn, "<=== (hint)");
3814 		return ret;
3815 	}
3816 
3817 	return 0;
3818 }
3819 
3820 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3821 {
3822 	struct instruction *insn;
3823 	struct insn_state state;
3824 	int warnings = 0;
3825 
3826 	if (!file->hints)
3827 		return 0;
3828 
3829 	init_insn_state(file, &state, sec);
3830 
3831 	if (sec) {
3832 		sec_for_each_insn(file, sec, insn)
3833 			warnings += validate_unwind_hint(file, insn, &state);
3834 	} else {
3835 		for_each_insn(file, insn)
3836 			warnings += validate_unwind_hint(file, insn, &state);
3837 	}
3838 
3839 	return warnings;
3840 }
3841 
3842 /*
3843  * Validate rethunk entry constraint: must untrain RET before the first RET.
3844  *
3845  * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3846  * before an actual RET instruction.
3847  */
3848 static int validate_unret(struct objtool_file *file, struct instruction *insn)
3849 {
3850 	struct instruction *next, *dest;
3851 	int ret;
3852 
3853 	for (;;) {
3854 		next = next_insn_to_validate(file, insn);
3855 
3856 		if (insn->visited & VISITED_UNRET)
3857 			return 0;
3858 
3859 		insn->visited |= VISITED_UNRET;
3860 
3861 		if (!insn->ignore_alts && insn->alts) {
3862 			struct alternative *alt;
3863 			bool skip_orig = false;
3864 
3865 			for (alt = insn->alts; alt; alt = alt->next) {
3866 				if (alt->skip_orig)
3867 					skip_orig = true;
3868 
3869 				ret = validate_unret(file, alt->insn);
3870 				if (ret) {
3871 					BT_INSN(insn, "(alt)");
3872 					return ret;
3873 				}
3874 			}
3875 
3876 			if (skip_orig)
3877 				return 0;
3878 		}
3879 
3880 		switch (insn->type) {
3881 
3882 		case INSN_CALL_DYNAMIC:
3883 		case INSN_JUMP_DYNAMIC:
3884 		case INSN_JUMP_DYNAMIC_CONDITIONAL:
3885 			WARN_INSN(insn, "early indirect call");
3886 			return 1;
3887 
3888 		case INSN_JUMP_UNCONDITIONAL:
3889 		case INSN_JUMP_CONDITIONAL:
3890 			if (!is_sibling_call(insn)) {
3891 				if (!insn->jump_dest) {
3892 					WARN_INSN(insn, "unresolved jump target after linking?!?");
3893 					return -1;
3894 				}
3895 				ret = validate_unret(file, insn->jump_dest);
3896 				if (ret) {
3897 					BT_INSN(insn, "(branch%s)",
3898 						insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3899 					return ret;
3900 				}
3901 
3902 				if (insn->type == INSN_JUMP_UNCONDITIONAL)
3903 					return 0;
3904 
3905 				break;
3906 			}
3907 
3908 			/* fallthrough */
3909 		case INSN_CALL:
3910 			dest = find_insn(file, insn_call_dest(insn)->sec,
3911 					 insn_call_dest(insn)->offset);
3912 			if (!dest) {
3913 				WARN("Unresolved function after linking!?: %s",
3914 				     insn_call_dest(insn)->name);
3915 				return -1;
3916 			}
3917 
3918 			ret = validate_unret(file, dest);
3919 			if (ret) {
3920 				BT_INSN(insn, "(call)");
3921 				return ret;
3922 			}
3923 			/*
3924 			 * If a call returns without error, it must have seen UNTRAIN_RET.
3925 			 * Therefore any non-error return is a success.
3926 			 */
3927 			return 0;
3928 
3929 		case INSN_RETURN:
3930 			WARN_INSN(insn, "RET before UNTRAIN");
3931 			return 1;
3932 
3933 		case INSN_NOP:
3934 			if (insn->retpoline_safe)
3935 				return 0;
3936 			break;
3937 
3938 		default:
3939 			break;
3940 		}
3941 
3942 		if (!next) {
3943 			WARN_INSN(insn, "teh end!");
3944 			return -1;
3945 		}
3946 		insn = next;
3947 	}
3948 
3949 	return 0;
3950 }
3951 
3952 /*
3953  * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3954  * VALIDATE_UNRET_END before RET.
3955  */
3956 static int validate_unrets(struct objtool_file *file)
3957 {
3958 	struct instruction *insn;
3959 	int ret, warnings = 0;
3960 
3961 	for_each_insn(file, insn) {
3962 		if (!insn->unret)
3963 			continue;
3964 
3965 		ret = validate_unret(file, insn);
3966 		if (ret < 0) {
3967 			WARN_INSN(insn, "Failed UNRET validation");
3968 			return ret;
3969 		}
3970 		warnings += ret;
3971 	}
3972 
3973 	return warnings;
3974 }
3975 
3976 static int validate_retpoline(struct objtool_file *file)
3977 {
3978 	struct instruction *insn;
3979 	int warnings = 0;
3980 
3981 	for_each_insn(file, insn) {
3982 		if (insn->type != INSN_JUMP_DYNAMIC &&
3983 		    insn->type != INSN_CALL_DYNAMIC &&
3984 		    insn->type != INSN_RETURN)
3985 			continue;
3986 
3987 		if (insn->retpoline_safe)
3988 			continue;
3989 
3990 		if (insn->sec->init)
3991 			continue;
3992 
3993 		if (insn->type == INSN_RETURN) {
3994 			if (opts.rethunk) {
3995 				WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
3996 			} else
3997 				continue;
3998 		} else {
3999 			WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
4000 				  insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4001 		}
4002 
4003 		warnings++;
4004 	}
4005 
4006 	return warnings;
4007 }
4008 
4009 static bool is_kasan_insn(struct instruction *insn)
4010 {
4011 	return (insn->type == INSN_CALL &&
4012 		!strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4013 }
4014 
4015 static bool is_ubsan_insn(struct instruction *insn)
4016 {
4017 	return (insn->type == INSN_CALL &&
4018 		!strcmp(insn_call_dest(insn)->name,
4019 			"__ubsan_handle_builtin_unreachable"));
4020 }
4021 
4022 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4023 {
4024 	int i;
4025 	struct instruction *prev_insn;
4026 
4027 	if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4028 		return true;
4029 
4030 	/*
4031 	 * Ignore alternative replacement instructions.  This can happen
4032 	 * when a whitelisted function uses one of the ALTERNATIVE macros.
4033 	 */
4034 	if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4035 	    !strcmp(insn->sec->name, ".altinstr_aux"))
4036 		return true;
4037 
4038 	/*
4039 	 * Whole archive runs might encounter dead code from weak symbols.
4040 	 * This is where the linker will have dropped the weak symbol in
4041 	 * favour of a regular symbol, but leaves the code in place.
4042 	 *
4043 	 * In this case we'll find a piece of code (whole function) that is not
4044 	 * covered by a !section symbol. Ignore them.
4045 	 */
4046 	if (opts.link && !insn_func(insn)) {
4047 		int size = find_symbol_hole_containing(insn->sec, insn->offset);
4048 		unsigned long end = insn->offset + size;
4049 
4050 		if (!size) /* not a hole */
4051 			return false;
4052 
4053 		if (size < 0) /* hole until the end */
4054 			return true;
4055 
4056 		sec_for_each_insn_continue(file, insn) {
4057 			/*
4058 			 * If we reach a visited instruction at or before the
4059 			 * end of the hole, ignore the unreachable.
4060 			 */
4061 			if (insn->visited)
4062 				return true;
4063 
4064 			if (insn->offset >= end)
4065 				break;
4066 
4067 			/*
4068 			 * If this hole jumps to a .cold function, mark it ignore too.
4069 			 */
4070 			if (insn->jump_dest && insn_func(insn->jump_dest) &&
4071 			    strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4072 				struct instruction *dest = insn->jump_dest;
4073 				func_for_each_insn(file, insn_func(dest), dest)
4074 					dest->ignore = true;
4075 			}
4076 		}
4077 
4078 		return false;
4079 	}
4080 
4081 	if (!insn_func(insn))
4082 		return false;
4083 
4084 	if (insn_func(insn)->static_call_tramp)
4085 		return true;
4086 
4087 	/*
4088 	 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4089 	 * __builtin_unreachable().  The BUG() macro has an unreachable() after
4090 	 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4091 	 * (or occasionally a JMP to UD2).
4092 	 *
4093 	 * It may also insert a UD2 after calling a __noreturn function.
4094 	 */
4095 	prev_insn = prev_insn_same_sec(file, insn);
4096 	if (prev_insn->dead_end &&
4097 	    (insn->type == INSN_BUG ||
4098 	     (insn->type == INSN_JUMP_UNCONDITIONAL &&
4099 	      insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4100 		return true;
4101 
4102 	/*
4103 	 * Check if this (or a subsequent) instruction is related to
4104 	 * CONFIG_UBSAN or CONFIG_KASAN.
4105 	 *
4106 	 * End the search at 5 instructions to avoid going into the weeds.
4107 	 */
4108 	for (i = 0; i < 5; i++) {
4109 
4110 		if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4111 			return true;
4112 
4113 		if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4114 			if (insn->jump_dest &&
4115 			    insn_func(insn->jump_dest) == insn_func(insn)) {
4116 				insn = insn->jump_dest;
4117 				continue;
4118 			}
4119 
4120 			break;
4121 		}
4122 
4123 		if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4124 			break;
4125 
4126 		insn = next_insn_same_sec(file, insn);
4127 	}
4128 
4129 	return false;
4130 }
4131 
4132 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4133 {
4134 	struct instruction *insn, *prev;
4135 	struct cfi_state *cfi;
4136 
4137 	insn = find_insn(file, func->sec, func->offset);
4138 	if (!insn)
4139 		return -1;
4140 
4141 	for (prev = prev_insn_same_sec(file, insn);
4142 	     prev;
4143 	     prev = prev_insn_same_sec(file, prev)) {
4144 		u64 offset;
4145 
4146 		if (prev->type != INSN_NOP)
4147 			return -1;
4148 
4149 		offset = func->offset - prev->offset;
4150 
4151 		if (offset > opts.prefix)
4152 			return -1;
4153 
4154 		if (offset < opts.prefix)
4155 			continue;
4156 
4157 		elf_create_prefix_symbol(file->elf, func, opts.prefix);
4158 		break;
4159 	}
4160 
4161 	if (!prev)
4162 		return -1;
4163 
4164 	if (!insn->cfi) {
4165 		/*
4166 		 * This can happen if stack validation isn't enabled or the
4167 		 * function is annotated with STACK_FRAME_NON_STANDARD.
4168 		 */
4169 		return 0;
4170 	}
4171 
4172 	/* Propagate insn->cfi to the prefix code */
4173 	cfi = cfi_hash_find_or_add(insn->cfi);
4174 	for (; prev != insn; prev = next_insn_same_sec(file, prev))
4175 		prev->cfi = cfi;
4176 
4177 	return 0;
4178 }
4179 
4180 static int add_prefix_symbols(struct objtool_file *file)
4181 {
4182 	struct section *sec;
4183 	struct symbol *func;
4184 
4185 	for_each_sec(file, sec) {
4186 		if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4187 			continue;
4188 
4189 		sec_for_each_sym(sec, func) {
4190 			if (func->type != STT_FUNC)
4191 				continue;
4192 
4193 			add_prefix_symbol(file, func);
4194 		}
4195 	}
4196 
4197 	return 0;
4198 }
4199 
4200 static int validate_symbol(struct objtool_file *file, struct section *sec,
4201 			   struct symbol *sym, struct insn_state *state)
4202 {
4203 	struct instruction *insn;
4204 	int ret;
4205 
4206 	if (!sym->len) {
4207 		WARN("%s() is missing an ELF size annotation", sym->name);
4208 		return 1;
4209 	}
4210 
4211 	if (sym->pfunc != sym || sym->alias != sym)
4212 		return 0;
4213 
4214 	insn = find_insn(file, sec, sym->offset);
4215 	if (!insn || insn->ignore || insn->visited)
4216 		return 0;
4217 
4218 	state->uaccess = sym->uaccess_safe;
4219 
4220 	ret = validate_branch(file, insn_func(insn), insn, *state);
4221 	if (ret)
4222 		BT_INSN(insn, "<=== (sym)");
4223 	return ret;
4224 }
4225 
4226 static int validate_section(struct objtool_file *file, struct section *sec)
4227 {
4228 	struct insn_state state;
4229 	struct symbol *func;
4230 	int warnings = 0;
4231 
4232 	sec_for_each_sym(sec, func) {
4233 		if (func->type != STT_FUNC)
4234 			continue;
4235 
4236 		init_insn_state(file, &state, sec);
4237 		set_func_state(&state.cfi);
4238 
4239 		warnings += validate_symbol(file, sec, func, &state);
4240 	}
4241 
4242 	return warnings;
4243 }
4244 
4245 static int validate_noinstr_sections(struct objtool_file *file)
4246 {
4247 	struct section *sec;
4248 	int warnings = 0;
4249 
4250 	sec = find_section_by_name(file->elf, ".noinstr.text");
4251 	if (sec) {
4252 		warnings += validate_section(file, sec);
4253 		warnings += validate_unwind_hints(file, sec);
4254 	}
4255 
4256 	sec = find_section_by_name(file->elf, ".entry.text");
4257 	if (sec) {
4258 		warnings += validate_section(file, sec);
4259 		warnings += validate_unwind_hints(file, sec);
4260 	}
4261 
4262 	sec = find_section_by_name(file->elf, ".cpuidle.text");
4263 	if (sec) {
4264 		warnings += validate_section(file, sec);
4265 		warnings += validate_unwind_hints(file, sec);
4266 	}
4267 
4268 	return warnings;
4269 }
4270 
4271 static int validate_functions(struct objtool_file *file)
4272 {
4273 	struct section *sec;
4274 	int warnings = 0;
4275 
4276 	for_each_sec(file, sec) {
4277 		if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4278 			continue;
4279 
4280 		warnings += validate_section(file, sec);
4281 	}
4282 
4283 	return warnings;
4284 }
4285 
4286 static void mark_endbr_used(struct instruction *insn)
4287 {
4288 	if (!list_empty(&insn->call_node))
4289 		list_del_init(&insn->call_node);
4290 }
4291 
4292 static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4293 {
4294 	struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4295 	struct instruction *first;
4296 
4297 	if (!sym)
4298 		return false;
4299 
4300 	first = find_insn(file, sym->sec, sym->offset);
4301 	if (!first)
4302 		return false;
4303 
4304 	if (first->type != INSN_ENDBR && !first->noendbr)
4305 		return false;
4306 
4307 	return insn->offset == sym->offset + sym->len;
4308 }
4309 
4310 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4311 {
4312 	struct instruction *dest;
4313 	struct reloc *reloc;
4314 	unsigned long off;
4315 	int warnings = 0;
4316 
4317 	/*
4318 	 * Looking for function pointer load relocations.  Ignore
4319 	 * direct/indirect branches:
4320 	 */
4321 	switch (insn->type) {
4322 	case INSN_CALL:
4323 	case INSN_CALL_DYNAMIC:
4324 	case INSN_JUMP_CONDITIONAL:
4325 	case INSN_JUMP_UNCONDITIONAL:
4326 	case INSN_JUMP_DYNAMIC:
4327 	case INSN_JUMP_DYNAMIC_CONDITIONAL:
4328 	case INSN_RETURN:
4329 	case INSN_NOP:
4330 		return 0;
4331 	default:
4332 		break;
4333 	}
4334 
4335 	for (reloc = insn_reloc(file, insn);
4336 	     reloc;
4337 	     reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4338 					      reloc_offset(reloc) + 1,
4339 					      (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4340 
4341 		/*
4342 		 * static_call_update() references the trampoline, which
4343 		 * doesn't have (or need) ENDBR.  Skip warning in that case.
4344 		 */
4345 		if (reloc->sym->static_call_tramp)
4346 			continue;
4347 
4348 		off = reloc->sym->offset;
4349 		if (reloc_type(reloc) == R_X86_64_PC32 ||
4350 		    reloc_type(reloc) == R_X86_64_PLT32)
4351 			off += arch_dest_reloc_offset(reloc_addend(reloc));
4352 		else
4353 			off += reloc_addend(reloc);
4354 
4355 		dest = find_insn(file, reloc->sym->sec, off);
4356 		if (!dest)
4357 			continue;
4358 
4359 		if (dest->type == INSN_ENDBR) {
4360 			mark_endbr_used(dest);
4361 			continue;
4362 		}
4363 
4364 		if (insn_func(dest) && insn_func(insn) &&
4365 		    insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4366 			/*
4367 			 * Anything from->to self is either _THIS_IP_ or
4368 			 * IRET-to-self.
4369 			 *
4370 			 * There is no sane way to annotate _THIS_IP_ since the
4371 			 * compiler treats the relocation as a constant and is
4372 			 * happy to fold in offsets, skewing any annotation we
4373 			 * do, leading to vast amounts of false-positives.
4374 			 *
4375 			 * There's also compiler generated _THIS_IP_ through
4376 			 * KCOV and such which we have no hope of annotating.
4377 			 *
4378 			 * As such, blanket accept self-references without
4379 			 * issue.
4380 			 */
4381 			continue;
4382 		}
4383 
4384 		/*
4385 		 * Accept anything ANNOTATE_NOENDBR.
4386 		 */
4387 		if (dest->noendbr)
4388 			continue;
4389 
4390 		/*
4391 		 * Accept if this is the instruction after a symbol
4392 		 * that is (no)endbr -- typical code-range usage.
4393 		 */
4394 		if (noendbr_range(file, dest))
4395 			continue;
4396 
4397 		WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4398 
4399 		warnings++;
4400 	}
4401 
4402 	return warnings;
4403 }
4404 
4405 static int validate_ibt_data_reloc(struct objtool_file *file,
4406 				   struct reloc *reloc)
4407 {
4408 	struct instruction *dest;
4409 
4410 	dest = find_insn(file, reloc->sym->sec,
4411 			 reloc->sym->offset + reloc_addend(reloc));
4412 	if (!dest)
4413 		return 0;
4414 
4415 	if (dest->type == INSN_ENDBR) {
4416 		mark_endbr_used(dest);
4417 		return 0;
4418 	}
4419 
4420 	if (dest->noendbr)
4421 		return 0;
4422 
4423 	WARN_FUNC("data relocation to !ENDBR: %s",
4424 		  reloc->sec->base, reloc_offset(reloc),
4425 		  offstr(dest->sec, dest->offset));
4426 
4427 	return 1;
4428 }
4429 
4430 /*
4431  * Validate IBT rules and remove used ENDBR instructions from the seal list.
4432  * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4433  * NOPs) later, in create_ibt_endbr_seal_sections().
4434  */
4435 static int validate_ibt(struct objtool_file *file)
4436 {
4437 	struct section *sec;
4438 	struct reloc *reloc;
4439 	struct instruction *insn;
4440 	int warnings = 0;
4441 
4442 	for_each_insn(file, insn)
4443 		warnings += validate_ibt_insn(file, insn);
4444 
4445 	for_each_sec(file, sec) {
4446 
4447 		/* Already done by validate_ibt_insn() */
4448 		if (sec->sh.sh_flags & SHF_EXECINSTR)
4449 			continue;
4450 
4451 		if (!sec->rsec)
4452 			continue;
4453 
4454 		/*
4455 		 * These sections can reference text addresses, but not with
4456 		 * the intent to indirect branch to them.
4457 		 */
4458 		if ((!strncmp(sec->name, ".discard", 8) &&
4459 		     strcmp(sec->name, ".discard.ibt_endbr_noseal"))	||
4460 		    !strncmp(sec->name, ".debug", 6)			||
4461 		    !strcmp(sec->name, ".altinstructions")		||
4462 		    !strcmp(sec->name, ".ibt_endbr_seal")		||
4463 		    !strcmp(sec->name, ".orc_unwind_ip")		||
4464 		    !strcmp(sec->name, ".parainstructions")		||
4465 		    !strcmp(sec->name, ".retpoline_sites")		||
4466 		    !strcmp(sec->name, ".smp_locks")			||
4467 		    !strcmp(sec->name, ".static_call_sites")		||
4468 		    !strcmp(sec->name, "_error_injection_whitelist")	||
4469 		    !strcmp(sec->name, "_kprobe_blacklist")		||
4470 		    !strcmp(sec->name, "__bug_table")			||
4471 		    !strcmp(sec->name, "__ex_table")			||
4472 		    !strcmp(sec->name, "__jump_table")			||
4473 		    !strcmp(sec->name, "__mcount_loc")			||
4474 		    !strcmp(sec->name, ".kcfi_traps")			||
4475 		    strstr(sec->name, "__patchable_function_entries"))
4476 			continue;
4477 
4478 		for_each_reloc(sec->rsec, reloc)
4479 			warnings += validate_ibt_data_reloc(file, reloc);
4480 	}
4481 
4482 	return warnings;
4483 }
4484 
4485 static int validate_sls(struct objtool_file *file)
4486 {
4487 	struct instruction *insn, *next_insn;
4488 	int warnings = 0;
4489 
4490 	for_each_insn(file, insn) {
4491 		next_insn = next_insn_same_sec(file, insn);
4492 
4493 		if (insn->retpoline_safe)
4494 			continue;
4495 
4496 		switch (insn->type) {
4497 		case INSN_RETURN:
4498 			if (!next_insn || next_insn->type != INSN_TRAP) {
4499 				WARN_INSN(insn, "missing int3 after ret");
4500 				warnings++;
4501 			}
4502 
4503 			break;
4504 		case INSN_JUMP_DYNAMIC:
4505 			if (!next_insn || next_insn->type != INSN_TRAP) {
4506 				WARN_INSN(insn, "missing int3 after indirect jump");
4507 				warnings++;
4508 			}
4509 			break;
4510 		default:
4511 			break;
4512 		}
4513 	}
4514 
4515 	return warnings;
4516 }
4517 
4518 static bool ignore_noreturn_call(struct instruction *insn)
4519 {
4520 	struct symbol *call_dest = insn_call_dest(insn);
4521 
4522 	/*
4523 	 * FIXME: hack, we need a real noreturn solution
4524 	 *
4525 	 * Problem is, exc_double_fault() may or may not return, depending on
4526 	 * whether CONFIG_X86_ESPFIX64 is set.  But objtool has no visibility
4527 	 * to the kernel config.
4528 	 *
4529 	 * Other potential ways to fix it:
4530 	 *
4531 	 *   - have compiler communicate __noreturn functions somehow
4532 	 *   - remove CONFIG_X86_ESPFIX64
4533 	 *   - read the .config file
4534 	 *   - add a cmdline option
4535 	 *   - create a generic objtool annotation format (vs a bunch of custom
4536 	 *     formats) and annotate it
4537 	 */
4538 	if (!strcmp(call_dest->name, "exc_double_fault")) {
4539 		/* prevent further unreachable warnings for the caller */
4540 		insn->sym->warned = 1;
4541 		return true;
4542 	}
4543 
4544 	return false;
4545 }
4546 
4547 static int validate_reachable_instructions(struct objtool_file *file)
4548 {
4549 	struct instruction *insn, *prev_insn;
4550 	struct symbol *call_dest;
4551 	int warnings = 0;
4552 
4553 	if (file->ignore_unreachables)
4554 		return 0;
4555 
4556 	for_each_insn(file, insn) {
4557 		if (insn->visited || ignore_unreachable_insn(file, insn))
4558 			continue;
4559 
4560 		prev_insn = prev_insn_same_sec(file, insn);
4561 		if (prev_insn && prev_insn->dead_end) {
4562 			call_dest = insn_call_dest(prev_insn);
4563 			if (call_dest && !ignore_noreturn_call(prev_insn)) {
4564 				WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4565 					  call_dest->name);
4566 				warnings++;
4567 				continue;
4568 			}
4569 		}
4570 
4571 		WARN_INSN(insn, "unreachable instruction");
4572 		warnings++;
4573 	}
4574 
4575 	return warnings;
4576 }
4577 
4578 /* 'funcs' is a space-separated list of function names */
4579 static int disas_funcs(const char *funcs)
4580 {
4581 	const char *objdump_str, *cross_compile;
4582 	int size, ret;
4583 	char *cmd;
4584 
4585 	cross_compile = getenv("CROSS_COMPILE");
4586 
4587 	objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4588 			"BEGIN { split(_funcs, funcs); }"
4589 			"/^$/ { func_match = 0; }"
4590 			"/<.*>:/ { "
4591 				"f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4592 				"for (i in funcs) {"
4593 					"if (funcs[i] == f) {"
4594 						"func_match = 1;"
4595 						"base = strtonum(\"0x\" $1);"
4596 						"break;"
4597 					"}"
4598 				"}"
4599 			"}"
4600 			"{"
4601 				"if (func_match) {"
4602 					"addr = strtonum(\"0x\" $1);"
4603 					"printf(\"%%04x \", addr - base);"
4604 					"print;"
4605 				"}"
4606 			"}' 1>&2";
4607 
4608 	/* fake snprintf() to calculate the size */
4609 	size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4610 	if (size <= 0) {
4611 		WARN("objdump string size calculation failed");
4612 		return -1;
4613 	}
4614 
4615 	cmd = malloc(size);
4616 
4617 	/* real snprintf() */
4618 	snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4619 	ret = system(cmd);
4620 	if (ret) {
4621 		WARN("disassembly failed: %d", ret);
4622 		return -1;
4623 	}
4624 
4625 	return 0;
4626 }
4627 
4628 static int disas_warned_funcs(struct objtool_file *file)
4629 {
4630 	struct symbol *sym;
4631 	char *funcs = NULL, *tmp;
4632 
4633 	for_each_sym(file, sym) {
4634 		if (sym->warned) {
4635 			if (!funcs) {
4636 				funcs = malloc(strlen(sym->name) + 1);
4637 				strcpy(funcs, sym->name);
4638 			} else {
4639 				tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4640 				sprintf(tmp, "%s %s", funcs, sym->name);
4641 				free(funcs);
4642 				funcs = tmp;
4643 			}
4644 		}
4645 	}
4646 
4647 	if (funcs)
4648 		disas_funcs(funcs);
4649 
4650 	return 0;
4651 }
4652 
4653 struct insn_chunk {
4654 	void *addr;
4655 	struct insn_chunk *next;
4656 };
4657 
4658 /*
4659  * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4660  * which can trigger more allocations for .debug_* sections whose data hasn't
4661  * been read yet.
4662  */
4663 static void free_insns(struct objtool_file *file)
4664 {
4665 	struct instruction *insn;
4666 	struct insn_chunk *chunks = NULL, *chunk;
4667 
4668 	for_each_insn(file, insn) {
4669 		if (!insn->idx) {
4670 			chunk = malloc(sizeof(*chunk));
4671 			chunk->addr = insn;
4672 			chunk->next = chunks;
4673 			chunks = chunk;
4674 		}
4675 	}
4676 
4677 	for (chunk = chunks; chunk; chunk = chunk->next)
4678 		free(chunk->addr);
4679 }
4680 
4681 int check(struct objtool_file *file)
4682 {
4683 	int ret, warnings = 0;
4684 
4685 	arch_initial_func_cfi_state(&initial_func_cfi);
4686 	init_cfi_state(&init_cfi);
4687 	init_cfi_state(&func_cfi);
4688 	set_func_state(&func_cfi);
4689 	init_cfi_state(&force_undefined_cfi);
4690 	force_undefined_cfi.force_undefined = true;
4691 
4692 	if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4693 		goto out;
4694 
4695 	cfi_hash_add(&init_cfi);
4696 	cfi_hash_add(&func_cfi);
4697 
4698 	ret = decode_sections(file);
4699 	if (ret < 0)
4700 		goto out;
4701 
4702 	warnings += ret;
4703 
4704 	if (!nr_insns)
4705 		goto out;
4706 
4707 	if (opts.retpoline) {
4708 		ret = validate_retpoline(file);
4709 		if (ret < 0)
4710 			return ret;
4711 		warnings += ret;
4712 	}
4713 
4714 	if (opts.stackval || opts.orc || opts.uaccess) {
4715 		ret = validate_functions(file);
4716 		if (ret < 0)
4717 			goto out;
4718 		warnings += ret;
4719 
4720 		ret = validate_unwind_hints(file, NULL);
4721 		if (ret < 0)
4722 			goto out;
4723 		warnings += ret;
4724 
4725 		if (!warnings) {
4726 			ret = validate_reachable_instructions(file);
4727 			if (ret < 0)
4728 				goto out;
4729 			warnings += ret;
4730 		}
4731 
4732 	} else if (opts.noinstr) {
4733 		ret = validate_noinstr_sections(file);
4734 		if (ret < 0)
4735 			goto out;
4736 		warnings += ret;
4737 	}
4738 
4739 	if (opts.unret) {
4740 		/*
4741 		 * Must be after validate_branch() and friends, it plays
4742 		 * further games with insn->visited.
4743 		 */
4744 		ret = validate_unrets(file);
4745 		if (ret < 0)
4746 			return ret;
4747 		warnings += ret;
4748 	}
4749 
4750 	if (opts.ibt) {
4751 		ret = validate_ibt(file);
4752 		if (ret < 0)
4753 			goto out;
4754 		warnings += ret;
4755 	}
4756 
4757 	if (opts.sls) {
4758 		ret = validate_sls(file);
4759 		if (ret < 0)
4760 			goto out;
4761 		warnings += ret;
4762 	}
4763 
4764 	if (opts.static_call) {
4765 		ret = create_static_call_sections(file);
4766 		if (ret < 0)
4767 			goto out;
4768 		warnings += ret;
4769 	}
4770 
4771 	if (opts.retpoline) {
4772 		ret = create_retpoline_sites_sections(file);
4773 		if (ret < 0)
4774 			goto out;
4775 		warnings += ret;
4776 	}
4777 
4778 	if (opts.cfi) {
4779 		ret = create_cfi_sections(file);
4780 		if (ret < 0)
4781 			goto out;
4782 		warnings += ret;
4783 	}
4784 
4785 	if (opts.rethunk) {
4786 		ret = create_return_sites_sections(file);
4787 		if (ret < 0)
4788 			goto out;
4789 		warnings += ret;
4790 
4791 		if (opts.hack_skylake) {
4792 			ret = create_direct_call_sections(file);
4793 			if (ret < 0)
4794 				goto out;
4795 			warnings += ret;
4796 		}
4797 	}
4798 
4799 	if (opts.mcount) {
4800 		ret = create_mcount_loc_sections(file);
4801 		if (ret < 0)
4802 			goto out;
4803 		warnings += ret;
4804 	}
4805 
4806 	if (opts.prefix) {
4807 		ret = add_prefix_symbols(file);
4808 		if (ret < 0)
4809 			return ret;
4810 		warnings += ret;
4811 	}
4812 
4813 	if (opts.ibt) {
4814 		ret = create_ibt_endbr_seal_sections(file);
4815 		if (ret < 0)
4816 			goto out;
4817 		warnings += ret;
4818 	}
4819 
4820 	if (opts.orc && nr_insns) {
4821 		ret = orc_create(file);
4822 		if (ret < 0)
4823 			goto out;
4824 		warnings += ret;
4825 	}
4826 
4827 	free_insns(file);
4828 
4829 	if (opts.verbose)
4830 		disas_warned_funcs(file);
4831 
4832 	if (opts.stats) {
4833 		printf("nr_insns_visited: %ld\n", nr_insns_visited);
4834 		printf("nr_cfi: %ld\n", nr_cfi);
4835 		printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4836 		printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4837 	}
4838 
4839 out:
4840 	/*
4841 	 *  For now, don't fail the kernel build on fatal warnings.  These
4842 	 *  errors are still fairly common due to the growing matrix of
4843 	 *  supported toolchains and their recent pace of change.
4844 	 */
4845 	return 0;
4846 }
4847