1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6 #include <string.h> 7 #include <stdlib.h> 8 #include <inttypes.h> 9 #include <sys/mman.h> 10 11 #include <arch/elf.h> 12 #include <objtool/builtin.h> 13 #include <objtool/cfi.h> 14 #include <objtool/arch.h> 15 #include <objtool/check.h> 16 #include <objtool/special.h> 17 #include <objtool/warn.h> 18 #include <objtool/endianness.h> 19 20 #include <linux/objtool.h> 21 #include <linux/hashtable.h> 22 #include <linux/kernel.h> 23 #include <linux/static_call_types.h> 24 25 struct alternative { 26 struct list_head list; 27 struct instruction *insn; 28 bool skip_orig; 29 }; 30 31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 32 33 static struct cfi_init_state initial_func_cfi; 34 static struct cfi_state init_cfi; 35 static struct cfi_state func_cfi; 36 37 struct instruction *find_insn(struct objtool_file *file, 38 struct section *sec, unsigned long offset) 39 { 40 struct instruction *insn; 41 42 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) { 43 if (insn->sec == sec && insn->offset == offset) 44 return insn; 45 } 46 47 return NULL; 48 } 49 50 static struct instruction *next_insn_same_sec(struct objtool_file *file, 51 struct instruction *insn) 52 { 53 struct instruction *next = list_next_entry(insn, list); 54 55 if (!next || &next->list == &file->insn_list || next->sec != insn->sec) 56 return NULL; 57 58 return next; 59 } 60 61 static struct instruction *next_insn_same_func(struct objtool_file *file, 62 struct instruction *insn) 63 { 64 struct instruction *next = list_next_entry(insn, list); 65 struct symbol *func = insn_func(insn); 66 67 if (!func) 68 return NULL; 69 70 if (&next->list != &file->insn_list && insn_func(next) == func) 71 return next; 72 73 /* Check if we're already in the subfunction: */ 74 if (func == func->cfunc) 75 return NULL; 76 77 /* Move to the subfunction: */ 78 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 79 } 80 81 static struct instruction *prev_insn_same_sym(struct objtool_file *file, 82 struct instruction *insn) 83 { 84 struct instruction *prev = list_prev_entry(insn, list); 85 86 if (&prev->list != &file->insn_list && insn_func(prev) == insn_func(insn)) 87 return prev; 88 89 return NULL; 90 } 91 92 #define func_for_each_insn(file, func, insn) \ 93 for (insn = find_insn(file, func->sec, func->offset); \ 94 insn; \ 95 insn = next_insn_same_func(file, insn)) 96 97 #define sym_for_each_insn(file, sym, insn) \ 98 for (insn = find_insn(file, sym->sec, sym->offset); \ 99 insn && &insn->list != &file->insn_list && \ 100 insn->sec == sym->sec && \ 101 insn->offset < sym->offset + sym->len; \ 102 insn = list_next_entry(insn, list)) 103 104 #define sym_for_each_insn_continue_reverse(file, sym, insn) \ 105 for (insn = list_prev_entry(insn, list); \ 106 &insn->list != &file->insn_list && \ 107 insn->sec == sym->sec && insn->offset >= sym->offset; \ 108 insn = list_prev_entry(insn, list)) 109 110 #define sec_for_each_insn_from(file, insn) \ 111 for (; insn; insn = next_insn_same_sec(file, insn)) 112 113 #define sec_for_each_insn_continue(file, insn) \ 114 for (insn = next_insn_same_sec(file, insn); insn; \ 115 insn = next_insn_same_sec(file, insn)) 116 117 static bool is_jump_table_jump(struct instruction *insn) 118 { 119 struct alt_group *alt_group = insn->alt_group; 120 121 if (insn->jump_table) 122 return true; 123 124 /* Retpoline alternative for a jump table? */ 125 return alt_group && alt_group->orig_group && 126 alt_group->orig_group->first_insn->jump_table; 127 } 128 129 static bool is_sibling_call(struct instruction *insn) 130 { 131 /* 132 * Assume only STT_FUNC calls have jump-tables. 133 */ 134 if (insn_func(insn)) { 135 /* An indirect jump is either a sibling call or a jump to a table. */ 136 if (insn->type == INSN_JUMP_DYNAMIC) 137 return !is_jump_table_jump(insn); 138 } 139 140 /* add_jump_destinations() sets insn->call_dest for sibling calls. */ 141 return (is_static_jump(insn) && insn->call_dest); 142 } 143 144 /* 145 * This checks to see if the given function is a "noreturn" function. 146 * 147 * For global functions which are outside the scope of this object file, we 148 * have to keep a manual list of them. 149 * 150 * For local functions, we have to detect them manually by simply looking for 151 * the lack of a return instruction. 152 */ 153 static bool __dead_end_function(struct objtool_file *file, struct symbol *func, 154 int recursion) 155 { 156 int i; 157 struct instruction *insn; 158 bool empty = true; 159 160 /* 161 * Unfortunately these have to be hard coded because the noreturn 162 * attribute isn't provided in ELF data. Keep 'em sorted. 163 */ 164 static const char * const global_noreturns[] = { 165 "__invalid_creds", 166 "__module_put_and_kthread_exit", 167 "__reiserfs_panic", 168 "__stack_chk_fail", 169 "__ubsan_handle_builtin_unreachable", 170 "cpu_bringup_and_idle", 171 "cpu_startup_entry", 172 "do_exit", 173 "do_group_exit", 174 "do_task_dead", 175 "ex_handler_msr_mce", 176 "fortify_panic", 177 "kthread_complete_and_exit", 178 "kthread_exit", 179 "kunit_try_catch_throw", 180 "lbug_with_loc", 181 "machine_real_restart", 182 "make_task_dead", 183 "panic", 184 "rewind_stack_and_make_dead", 185 "sev_es_terminate", 186 "snp_abort", 187 "stop_this_cpu", 188 "usercopy_abort", 189 "xen_start_kernel", 190 }; 191 192 if (!func) 193 return false; 194 195 if (func->bind == STB_WEAK) 196 return false; 197 198 if (func->bind == STB_GLOBAL) 199 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) 200 if (!strcmp(func->name, global_noreturns[i])) 201 return true; 202 203 if (!func->len) 204 return false; 205 206 insn = find_insn(file, func->sec, func->offset); 207 if (!insn || !insn_func(insn)) 208 return false; 209 210 func_for_each_insn(file, func, insn) { 211 empty = false; 212 213 if (insn->type == INSN_RETURN) 214 return false; 215 } 216 217 if (empty) 218 return false; 219 220 /* 221 * A function can have a sibling call instead of a return. In that 222 * case, the function's dead-end status depends on whether the target 223 * of the sibling call returns. 224 */ 225 func_for_each_insn(file, func, insn) { 226 if (is_sibling_call(insn)) { 227 struct instruction *dest = insn->jump_dest; 228 229 if (!dest) 230 /* sibling call to another file */ 231 return false; 232 233 /* local sibling call */ 234 if (recursion == 5) { 235 /* 236 * Infinite recursion: two functions have 237 * sibling calls to each other. This is a very 238 * rare case. It means they aren't dead ends. 239 */ 240 return false; 241 } 242 243 return __dead_end_function(file, insn_func(dest), recursion+1); 244 } 245 } 246 247 return true; 248 } 249 250 static bool dead_end_function(struct objtool_file *file, struct symbol *func) 251 { 252 return __dead_end_function(file, func, 0); 253 } 254 255 static void init_cfi_state(struct cfi_state *cfi) 256 { 257 int i; 258 259 for (i = 0; i < CFI_NUM_REGS; i++) { 260 cfi->regs[i].base = CFI_UNDEFINED; 261 cfi->vals[i].base = CFI_UNDEFINED; 262 } 263 cfi->cfa.base = CFI_UNDEFINED; 264 cfi->drap_reg = CFI_UNDEFINED; 265 cfi->drap_offset = -1; 266 } 267 268 static void init_insn_state(struct objtool_file *file, struct insn_state *state, 269 struct section *sec) 270 { 271 memset(state, 0, sizeof(*state)); 272 init_cfi_state(&state->cfi); 273 274 /* 275 * We need the full vmlinux for noinstr validation, otherwise we can 276 * not correctly determine insn->call_dest->sec (external symbols do 277 * not have a section). 278 */ 279 if (opts.link && opts.noinstr && sec) 280 state->noinstr = sec->noinstr; 281 } 282 283 static struct cfi_state *cfi_alloc(void) 284 { 285 struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1); 286 if (!cfi) { 287 WARN("calloc failed"); 288 exit(1); 289 } 290 nr_cfi++; 291 return cfi; 292 } 293 294 static int cfi_bits; 295 static struct hlist_head *cfi_hash; 296 297 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2) 298 { 299 return memcmp((void *)cfi1 + sizeof(cfi1->hash), 300 (void *)cfi2 + sizeof(cfi2->hash), 301 sizeof(struct cfi_state) - sizeof(struct hlist_node)); 302 } 303 304 static inline u32 cfi_key(struct cfi_state *cfi) 305 { 306 return jhash((void *)cfi + sizeof(cfi->hash), 307 sizeof(*cfi) - sizeof(cfi->hash), 0); 308 } 309 310 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi) 311 { 312 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 313 struct cfi_state *obj; 314 315 hlist_for_each_entry(obj, head, hash) { 316 if (!cficmp(cfi, obj)) { 317 nr_cfi_cache++; 318 return obj; 319 } 320 } 321 322 obj = cfi_alloc(); 323 *obj = *cfi; 324 hlist_add_head(&obj->hash, head); 325 326 return obj; 327 } 328 329 static void cfi_hash_add(struct cfi_state *cfi) 330 { 331 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 332 333 hlist_add_head(&cfi->hash, head); 334 } 335 336 static void *cfi_hash_alloc(unsigned long size) 337 { 338 cfi_bits = max(10, ilog2(size)); 339 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits, 340 PROT_READ|PROT_WRITE, 341 MAP_PRIVATE|MAP_ANON, -1, 0); 342 if (cfi_hash == (void *)-1L) { 343 WARN("mmap fail cfi_hash"); 344 cfi_hash = NULL; 345 } else if (opts.stats) { 346 printf("cfi_bits: %d\n", cfi_bits); 347 } 348 349 return cfi_hash; 350 } 351 352 static unsigned long nr_insns; 353 static unsigned long nr_insns_visited; 354 355 /* 356 * Call the arch-specific instruction decoder for all the instructions and add 357 * them to the global instruction list. 358 */ 359 static int decode_instructions(struct objtool_file *file) 360 { 361 struct section *sec; 362 struct symbol *func; 363 unsigned long offset; 364 struct instruction *insn; 365 int ret; 366 367 for_each_sec(file, sec) { 368 369 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 370 continue; 371 372 if (strcmp(sec->name, ".altinstr_replacement") && 373 strcmp(sec->name, ".altinstr_aux") && 374 strncmp(sec->name, ".discard.", 9)) 375 sec->text = true; 376 377 if (!strcmp(sec->name, ".noinstr.text") || 378 !strcmp(sec->name, ".entry.text") || 379 !strncmp(sec->name, ".text.__x86.", 12)) 380 sec->noinstr = true; 381 382 /* 383 * .init.text code is ran before userspace and thus doesn't 384 * strictly need retpolines, except for modules which are 385 * loaded late, they very much do need retpoline in their 386 * .init.text 387 */ 388 if (!strcmp(sec->name, ".init.text") && !opts.module) 389 sec->init = true; 390 391 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) { 392 insn = malloc(sizeof(*insn)); 393 if (!insn) { 394 WARN("malloc failed"); 395 return -1; 396 } 397 memset(insn, 0, sizeof(*insn)); 398 INIT_LIST_HEAD(&insn->alts); 399 INIT_LIST_HEAD(&insn->stack_ops); 400 INIT_LIST_HEAD(&insn->call_node); 401 402 insn->sec = sec; 403 insn->offset = offset; 404 405 ret = arch_decode_instruction(file, sec, offset, 406 sec->sh.sh_size - offset, 407 &insn->len, &insn->type, 408 &insn->immediate, 409 &insn->stack_ops); 410 if (ret) 411 goto err; 412 413 /* 414 * By default, "ud2" is a dead end unless otherwise 415 * annotated, because GCC 7 inserts it for certain 416 * divide-by-zero cases. 417 */ 418 if (insn->type == INSN_BUG) 419 insn->dead_end = true; 420 421 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset)); 422 list_add_tail(&insn->list, &file->insn_list); 423 nr_insns++; 424 } 425 426 list_for_each_entry(func, &sec->symbol_list, list) { 427 if (func->type != STT_NOTYPE && func->type != STT_FUNC) 428 continue; 429 430 if (func->offset == sec->sh.sh_size) { 431 /* Heuristic: likely an "end" symbol */ 432 if (func->type == STT_NOTYPE) 433 continue; 434 WARN("%s(): STT_FUNC at end of section", 435 func->name); 436 return -1; 437 } 438 439 if (func->return_thunk || func->alias != func) 440 continue; 441 442 if (!find_insn(file, sec, func->offset)) { 443 WARN("%s(): can't find starting instruction", 444 func->name); 445 return -1; 446 } 447 448 sym_for_each_insn(file, func, insn) { 449 insn->sym = func; 450 if (func->type == STT_FUNC && 451 insn->type == INSN_ENDBR && 452 list_empty(&insn->call_node)) { 453 if (insn->offset == func->offset) { 454 list_add_tail(&insn->call_node, &file->endbr_list); 455 file->nr_endbr++; 456 } else { 457 file->nr_endbr_int++; 458 } 459 } 460 } 461 } 462 } 463 464 if (opts.stats) 465 printf("nr_insns: %lu\n", nr_insns); 466 467 return 0; 468 469 err: 470 free(insn); 471 return ret; 472 } 473 474 /* 475 * Read the pv_ops[] .data table to find the static initialized values. 476 */ 477 static int add_pv_ops(struct objtool_file *file, const char *symname) 478 { 479 struct symbol *sym, *func; 480 unsigned long off, end; 481 struct reloc *rel; 482 int idx; 483 484 sym = find_symbol_by_name(file->elf, symname); 485 if (!sym) 486 return 0; 487 488 off = sym->offset; 489 end = off + sym->len; 490 for (;;) { 491 rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 492 if (!rel) 493 break; 494 495 func = rel->sym; 496 if (func->type == STT_SECTION) 497 func = find_symbol_by_offset(rel->sym->sec, rel->addend); 498 499 idx = (rel->offset - sym->offset) / sizeof(unsigned long); 500 501 objtool_pv_add(file, idx, func); 502 503 off = rel->offset + 1; 504 if (off > end) 505 break; 506 } 507 508 return 0; 509 } 510 511 /* 512 * Allocate and initialize file->pv_ops[]. 513 */ 514 static int init_pv_ops(struct objtool_file *file) 515 { 516 static const char *pv_ops_tables[] = { 517 "pv_ops", 518 "xen_cpu_ops", 519 "xen_irq_ops", 520 "xen_mmu_ops", 521 NULL, 522 }; 523 const char *pv_ops; 524 struct symbol *sym; 525 int idx, nr; 526 527 if (!opts.noinstr) 528 return 0; 529 530 file->pv_ops = NULL; 531 532 sym = find_symbol_by_name(file->elf, "pv_ops"); 533 if (!sym) 534 return 0; 535 536 nr = sym->len / sizeof(unsigned long); 537 file->pv_ops = calloc(sizeof(struct pv_state), nr); 538 if (!file->pv_ops) 539 return -1; 540 541 for (idx = 0; idx < nr; idx++) 542 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 543 544 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) 545 add_pv_ops(file, pv_ops); 546 547 return 0; 548 } 549 550 static struct instruction *find_last_insn(struct objtool_file *file, 551 struct section *sec) 552 { 553 struct instruction *insn = NULL; 554 unsigned int offset; 555 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0; 556 557 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--) 558 insn = find_insn(file, sec, offset); 559 560 return insn; 561 } 562 563 /* 564 * Mark "ud2" instructions and manually annotated dead ends. 565 */ 566 static int add_dead_ends(struct objtool_file *file) 567 { 568 struct section *sec; 569 struct reloc *reloc; 570 struct instruction *insn; 571 572 /* 573 * Check for manually annotated dead ends. 574 */ 575 sec = find_section_by_name(file->elf, ".rela.discard.unreachable"); 576 if (!sec) 577 goto reachable; 578 579 list_for_each_entry(reloc, &sec->reloc_list, list) { 580 if (reloc->sym->type != STT_SECTION) { 581 WARN("unexpected relocation symbol type in %s", sec->name); 582 return -1; 583 } 584 insn = find_insn(file, reloc->sym->sec, reloc->addend); 585 if (insn) 586 insn = list_prev_entry(insn, list); 587 else if (reloc->addend == reloc->sym->sec->sh.sh_size) { 588 insn = find_last_insn(file, reloc->sym->sec); 589 if (!insn) { 590 WARN("can't find unreachable insn at %s+0x%" PRIx64, 591 reloc->sym->sec->name, reloc->addend); 592 return -1; 593 } 594 } else { 595 WARN("can't find unreachable insn at %s+0x%" PRIx64, 596 reloc->sym->sec->name, reloc->addend); 597 return -1; 598 } 599 600 insn->dead_end = true; 601 } 602 603 reachable: 604 /* 605 * These manually annotated reachable checks are needed for GCC 4.4, 606 * where the Linux unreachable() macro isn't supported. In that case 607 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's 608 * not a dead end. 609 */ 610 sec = find_section_by_name(file->elf, ".rela.discard.reachable"); 611 if (!sec) 612 return 0; 613 614 list_for_each_entry(reloc, &sec->reloc_list, list) { 615 if (reloc->sym->type != STT_SECTION) { 616 WARN("unexpected relocation symbol type in %s", sec->name); 617 return -1; 618 } 619 insn = find_insn(file, reloc->sym->sec, reloc->addend); 620 if (insn) 621 insn = list_prev_entry(insn, list); 622 else if (reloc->addend == reloc->sym->sec->sh.sh_size) { 623 insn = find_last_insn(file, reloc->sym->sec); 624 if (!insn) { 625 WARN("can't find reachable insn at %s+0x%" PRIx64, 626 reloc->sym->sec->name, reloc->addend); 627 return -1; 628 } 629 } else { 630 WARN("can't find reachable insn at %s+0x%" PRIx64, 631 reloc->sym->sec->name, reloc->addend); 632 return -1; 633 } 634 635 insn->dead_end = false; 636 } 637 638 return 0; 639 } 640 641 static int create_static_call_sections(struct objtool_file *file) 642 { 643 struct section *sec; 644 struct static_call_site *site; 645 struct instruction *insn; 646 struct symbol *key_sym; 647 char *key_name, *tmp; 648 int idx; 649 650 sec = find_section_by_name(file->elf, ".static_call_sites"); 651 if (sec) { 652 INIT_LIST_HEAD(&file->static_call_list); 653 WARN("file already has .static_call_sites section, skipping"); 654 return 0; 655 } 656 657 if (list_empty(&file->static_call_list)) 658 return 0; 659 660 idx = 0; 661 list_for_each_entry(insn, &file->static_call_list, call_node) 662 idx++; 663 664 sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE, 665 sizeof(struct static_call_site), idx); 666 if (!sec) 667 return -1; 668 669 idx = 0; 670 list_for_each_entry(insn, &file->static_call_list, call_node) { 671 672 site = (struct static_call_site *)sec->data->d_buf + idx; 673 memset(site, 0, sizeof(struct static_call_site)); 674 675 /* populate reloc for 'addr' */ 676 if (elf_add_reloc_to_insn(file->elf, sec, 677 idx * sizeof(struct static_call_site), 678 R_X86_64_PC32, 679 insn->sec, insn->offset)) 680 return -1; 681 682 /* find key symbol */ 683 key_name = strdup(insn->call_dest->name); 684 if (!key_name) { 685 perror("strdup"); 686 return -1; 687 } 688 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR, 689 STATIC_CALL_TRAMP_PREFIX_LEN)) { 690 WARN("static_call: trampoline name malformed: %s", key_name); 691 return -1; 692 } 693 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN; 694 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN); 695 696 key_sym = find_symbol_by_name(file->elf, tmp); 697 if (!key_sym) { 698 if (!opts.module) { 699 WARN("static_call: can't find static_call_key symbol: %s", tmp); 700 return -1; 701 } 702 703 /* 704 * For modules(), the key might not be exported, which 705 * means the module can make static calls but isn't 706 * allowed to change them. 707 * 708 * In that case we temporarily set the key to be the 709 * trampoline address. This is fixed up in 710 * static_call_add_module(). 711 */ 712 key_sym = insn->call_dest; 713 } 714 free(key_name); 715 716 /* populate reloc for 'key' */ 717 if (elf_add_reloc(file->elf, sec, 718 idx * sizeof(struct static_call_site) + 4, 719 R_X86_64_PC32, key_sym, 720 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL)) 721 return -1; 722 723 idx++; 724 } 725 726 return 0; 727 } 728 729 static int create_retpoline_sites_sections(struct objtool_file *file) 730 { 731 struct instruction *insn; 732 struct section *sec; 733 int idx; 734 735 sec = find_section_by_name(file->elf, ".retpoline_sites"); 736 if (sec) { 737 WARN("file already has .retpoline_sites, skipping"); 738 return 0; 739 } 740 741 idx = 0; 742 list_for_each_entry(insn, &file->retpoline_call_list, call_node) 743 idx++; 744 745 if (!idx) 746 return 0; 747 748 sec = elf_create_section(file->elf, ".retpoline_sites", 0, 749 sizeof(int), idx); 750 if (!sec) { 751 WARN("elf_create_section: .retpoline_sites"); 752 return -1; 753 } 754 755 idx = 0; 756 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 757 758 int *site = (int *)sec->data->d_buf + idx; 759 *site = 0; 760 761 if (elf_add_reloc_to_insn(file->elf, sec, 762 idx * sizeof(int), 763 R_X86_64_PC32, 764 insn->sec, insn->offset)) { 765 WARN("elf_add_reloc_to_insn: .retpoline_sites"); 766 return -1; 767 } 768 769 idx++; 770 } 771 772 return 0; 773 } 774 775 static int create_return_sites_sections(struct objtool_file *file) 776 { 777 struct instruction *insn; 778 struct section *sec; 779 int idx; 780 781 sec = find_section_by_name(file->elf, ".return_sites"); 782 if (sec) { 783 WARN("file already has .return_sites, skipping"); 784 return 0; 785 } 786 787 idx = 0; 788 list_for_each_entry(insn, &file->return_thunk_list, call_node) 789 idx++; 790 791 if (!idx) 792 return 0; 793 794 sec = elf_create_section(file->elf, ".return_sites", 0, 795 sizeof(int), idx); 796 if (!sec) { 797 WARN("elf_create_section: .return_sites"); 798 return -1; 799 } 800 801 idx = 0; 802 list_for_each_entry(insn, &file->return_thunk_list, call_node) { 803 804 int *site = (int *)sec->data->d_buf + idx; 805 *site = 0; 806 807 if (elf_add_reloc_to_insn(file->elf, sec, 808 idx * sizeof(int), 809 R_X86_64_PC32, 810 insn->sec, insn->offset)) { 811 WARN("elf_add_reloc_to_insn: .return_sites"); 812 return -1; 813 } 814 815 idx++; 816 } 817 818 return 0; 819 } 820 821 static int create_ibt_endbr_seal_sections(struct objtool_file *file) 822 { 823 struct instruction *insn; 824 struct section *sec; 825 int idx; 826 827 sec = find_section_by_name(file->elf, ".ibt_endbr_seal"); 828 if (sec) { 829 WARN("file already has .ibt_endbr_seal, skipping"); 830 return 0; 831 } 832 833 idx = 0; 834 list_for_each_entry(insn, &file->endbr_list, call_node) 835 idx++; 836 837 if (opts.stats) { 838 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); 839 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); 840 printf("ibt: superfluous ENDBR: %d\n", idx); 841 } 842 843 if (!idx) 844 return 0; 845 846 sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0, 847 sizeof(int), idx); 848 if (!sec) { 849 WARN("elf_create_section: .ibt_endbr_seal"); 850 return -1; 851 } 852 853 idx = 0; 854 list_for_each_entry(insn, &file->endbr_list, call_node) { 855 856 int *site = (int *)sec->data->d_buf + idx; 857 *site = 0; 858 859 if (elf_add_reloc_to_insn(file->elf, sec, 860 idx * sizeof(int), 861 R_X86_64_PC32, 862 insn->sec, insn->offset)) { 863 WARN("elf_add_reloc_to_insn: .ibt_endbr_seal"); 864 return -1; 865 } 866 867 idx++; 868 } 869 870 return 0; 871 } 872 873 static int create_cfi_sections(struct objtool_file *file) 874 { 875 struct section *sec, *s; 876 struct symbol *sym; 877 unsigned int *loc; 878 int idx; 879 880 sec = find_section_by_name(file->elf, ".cfi_sites"); 881 if (sec) { 882 INIT_LIST_HEAD(&file->call_list); 883 WARN("file already has .cfi_sites section, skipping"); 884 return 0; 885 } 886 887 idx = 0; 888 for_each_sec(file, s) { 889 if (!s->text) 890 continue; 891 892 list_for_each_entry(sym, &s->symbol_list, list) { 893 if (sym->type != STT_FUNC) 894 continue; 895 896 if (strncmp(sym->name, "__cfi_", 6)) 897 continue; 898 899 idx++; 900 } 901 } 902 903 sec = elf_create_section(file->elf, ".cfi_sites", 0, sizeof(unsigned int), idx); 904 if (!sec) 905 return -1; 906 907 idx = 0; 908 for_each_sec(file, s) { 909 if (!s->text) 910 continue; 911 912 list_for_each_entry(sym, &s->symbol_list, list) { 913 if (sym->type != STT_FUNC) 914 continue; 915 916 if (strncmp(sym->name, "__cfi_", 6)) 917 continue; 918 919 loc = (unsigned int *)sec->data->d_buf + idx; 920 memset(loc, 0, sizeof(unsigned int)); 921 922 if (elf_add_reloc_to_insn(file->elf, sec, 923 idx * sizeof(unsigned int), 924 R_X86_64_PC32, 925 s, sym->offset)) 926 return -1; 927 928 idx++; 929 } 930 } 931 932 return 0; 933 } 934 935 static int create_mcount_loc_sections(struct objtool_file *file) 936 { 937 int addrsize = elf_class_addrsize(file->elf); 938 struct instruction *insn; 939 struct section *sec; 940 int idx; 941 942 sec = find_section_by_name(file->elf, "__mcount_loc"); 943 if (sec) { 944 INIT_LIST_HEAD(&file->mcount_loc_list); 945 WARN("file already has __mcount_loc section, skipping"); 946 return 0; 947 } 948 949 if (list_empty(&file->mcount_loc_list)) 950 return 0; 951 952 idx = 0; 953 list_for_each_entry(insn, &file->mcount_loc_list, call_node) 954 idx++; 955 956 sec = elf_create_section(file->elf, "__mcount_loc", 0, addrsize, idx); 957 if (!sec) 958 return -1; 959 960 sec->sh.sh_addralign = addrsize; 961 962 idx = 0; 963 list_for_each_entry(insn, &file->mcount_loc_list, call_node) { 964 void *loc; 965 966 loc = sec->data->d_buf + idx; 967 memset(loc, 0, addrsize); 968 969 if (elf_add_reloc_to_insn(file->elf, sec, idx, 970 addrsize == sizeof(u64) ? R_ABS64 : R_ABS32, 971 insn->sec, insn->offset)) 972 return -1; 973 974 idx += addrsize; 975 } 976 977 return 0; 978 } 979 980 static int create_direct_call_sections(struct objtool_file *file) 981 { 982 struct instruction *insn; 983 struct section *sec; 984 unsigned int *loc; 985 int idx; 986 987 sec = find_section_by_name(file->elf, ".call_sites"); 988 if (sec) { 989 INIT_LIST_HEAD(&file->call_list); 990 WARN("file already has .call_sites section, skipping"); 991 return 0; 992 } 993 994 if (list_empty(&file->call_list)) 995 return 0; 996 997 idx = 0; 998 list_for_each_entry(insn, &file->call_list, call_node) 999 idx++; 1000 1001 sec = elf_create_section(file->elf, ".call_sites", 0, sizeof(unsigned int), idx); 1002 if (!sec) 1003 return -1; 1004 1005 idx = 0; 1006 list_for_each_entry(insn, &file->call_list, call_node) { 1007 1008 loc = (unsigned int *)sec->data->d_buf + idx; 1009 memset(loc, 0, sizeof(unsigned int)); 1010 1011 if (elf_add_reloc_to_insn(file->elf, sec, 1012 idx * sizeof(unsigned int), 1013 R_X86_64_PC32, 1014 insn->sec, insn->offset)) 1015 return -1; 1016 1017 idx++; 1018 } 1019 1020 return 0; 1021 } 1022 1023 /* 1024 * Warnings shouldn't be reported for ignored functions. 1025 */ 1026 static void add_ignores(struct objtool_file *file) 1027 { 1028 struct instruction *insn; 1029 struct section *sec; 1030 struct symbol *func; 1031 struct reloc *reloc; 1032 1033 sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard"); 1034 if (!sec) 1035 return; 1036 1037 list_for_each_entry(reloc, &sec->reloc_list, list) { 1038 switch (reloc->sym->type) { 1039 case STT_FUNC: 1040 func = reloc->sym; 1041 break; 1042 1043 case STT_SECTION: 1044 func = find_func_by_offset(reloc->sym->sec, reloc->addend); 1045 if (!func) 1046 continue; 1047 break; 1048 1049 default: 1050 WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type); 1051 continue; 1052 } 1053 1054 func_for_each_insn(file, func, insn) 1055 insn->ignore = true; 1056 } 1057 } 1058 1059 /* 1060 * This is a whitelist of functions that is allowed to be called with AC set. 1061 * The list is meant to be minimal and only contains compiler instrumentation 1062 * ABI and a few functions used to implement *_{to,from}_user() functions. 1063 * 1064 * These functions must not directly change AC, but may PUSHF/POPF. 1065 */ 1066 static const char *uaccess_safe_builtin[] = { 1067 /* KASAN */ 1068 "kasan_report", 1069 "kasan_check_range", 1070 /* KASAN out-of-line */ 1071 "__asan_loadN_noabort", 1072 "__asan_load1_noabort", 1073 "__asan_load2_noabort", 1074 "__asan_load4_noabort", 1075 "__asan_load8_noabort", 1076 "__asan_load16_noabort", 1077 "__asan_storeN_noabort", 1078 "__asan_store1_noabort", 1079 "__asan_store2_noabort", 1080 "__asan_store4_noabort", 1081 "__asan_store8_noabort", 1082 "__asan_store16_noabort", 1083 "__kasan_check_read", 1084 "__kasan_check_write", 1085 /* KASAN in-line */ 1086 "__asan_report_load_n_noabort", 1087 "__asan_report_load1_noabort", 1088 "__asan_report_load2_noabort", 1089 "__asan_report_load4_noabort", 1090 "__asan_report_load8_noabort", 1091 "__asan_report_load16_noabort", 1092 "__asan_report_store_n_noabort", 1093 "__asan_report_store1_noabort", 1094 "__asan_report_store2_noabort", 1095 "__asan_report_store4_noabort", 1096 "__asan_report_store8_noabort", 1097 "__asan_report_store16_noabort", 1098 /* KCSAN */ 1099 "__kcsan_check_access", 1100 "__kcsan_mb", 1101 "__kcsan_wmb", 1102 "__kcsan_rmb", 1103 "__kcsan_release", 1104 "kcsan_found_watchpoint", 1105 "kcsan_setup_watchpoint", 1106 "kcsan_check_scoped_accesses", 1107 "kcsan_disable_current", 1108 "kcsan_enable_current_nowarn", 1109 /* KCSAN/TSAN */ 1110 "__tsan_func_entry", 1111 "__tsan_func_exit", 1112 "__tsan_read_range", 1113 "__tsan_write_range", 1114 "__tsan_read1", 1115 "__tsan_read2", 1116 "__tsan_read4", 1117 "__tsan_read8", 1118 "__tsan_read16", 1119 "__tsan_write1", 1120 "__tsan_write2", 1121 "__tsan_write4", 1122 "__tsan_write8", 1123 "__tsan_write16", 1124 "__tsan_read_write1", 1125 "__tsan_read_write2", 1126 "__tsan_read_write4", 1127 "__tsan_read_write8", 1128 "__tsan_read_write16", 1129 "__tsan_volatile_read1", 1130 "__tsan_volatile_read2", 1131 "__tsan_volatile_read4", 1132 "__tsan_volatile_read8", 1133 "__tsan_volatile_read16", 1134 "__tsan_volatile_write1", 1135 "__tsan_volatile_write2", 1136 "__tsan_volatile_write4", 1137 "__tsan_volatile_write8", 1138 "__tsan_volatile_write16", 1139 "__tsan_atomic8_load", 1140 "__tsan_atomic16_load", 1141 "__tsan_atomic32_load", 1142 "__tsan_atomic64_load", 1143 "__tsan_atomic8_store", 1144 "__tsan_atomic16_store", 1145 "__tsan_atomic32_store", 1146 "__tsan_atomic64_store", 1147 "__tsan_atomic8_exchange", 1148 "__tsan_atomic16_exchange", 1149 "__tsan_atomic32_exchange", 1150 "__tsan_atomic64_exchange", 1151 "__tsan_atomic8_fetch_add", 1152 "__tsan_atomic16_fetch_add", 1153 "__tsan_atomic32_fetch_add", 1154 "__tsan_atomic64_fetch_add", 1155 "__tsan_atomic8_fetch_sub", 1156 "__tsan_atomic16_fetch_sub", 1157 "__tsan_atomic32_fetch_sub", 1158 "__tsan_atomic64_fetch_sub", 1159 "__tsan_atomic8_fetch_and", 1160 "__tsan_atomic16_fetch_and", 1161 "__tsan_atomic32_fetch_and", 1162 "__tsan_atomic64_fetch_and", 1163 "__tsan_atomic8_fetch_or", 1164 "__tsan_atomic16_fetch_or", 1165 "__tsan_atomic32_fetch_or", 1166 "__tsan_atomic64_fetch_or", 1167 "__tsan_atomic8_fetch_xor", 1168 "__tsan_atomic16_fetch_xor", 1169 "__tsan_atomic32_fetch_xor", 1170 "__tsan_atomic64_fetch_xor", 1171 "__tsan_atomic8_fetch_nand", 1172 "__tsan_atomic16_fetch_nand", 1173 "__tsan_atomic32_fetch_nand", 1174 "__tsan_atomic64_fetch_nand", 1175 "__tsan_atomic8_compare_exchange_strong", 1176 "__tsan_atomic16_compare_exchange_strong", 1177 "__tsan_atomic32_compare_exchange_strong", 1178 "__tsan_atomic64_compare_exchange_strong", 1179 "__tsan_atomic8_compare_exchange_weak", 1180 "__tsan_atomic16_compare_exchange_weak", 1181 "__tsan_atomic32_compare_exchange_weak", 1182 "__tsan_atomic64_compare_exchange_weak", 1183 "__tsan_atomic8_compare_exchange_val", 1184 "__tsan_atomic16_compare_exchange_val", 1185 "__tsan_atomic32_compare_exchange_val", 1186 "__tsan_atomic64_compare_exchange_val", 1187 "__tsan_atomic_thread_fence", 1188 "__tsan_atomic_signal_fence", 1189 /* KCOV */ 1190 "write_comp_data", 1191 "check_kcov_mode", 1192 "__sanitizer_cov_trace_pc", 1193 "__sanitizer_cov_trace_const_cmp1", 1194 "__sanitizer_cov_trace_const_cmp2", 1195 "__sanitizer_cov_trace_const_cmp4", 1196 "__sanitizer_cov_trace_const_cmp8", 1197 "__sanitizer_cov_trace_cmp1", 1198 "__sanitizer_cov_trace_cmp2", 1199 "__sanitizer_cov_trace_cmp4", 1200 "__sanitizer_cov_trace_cmp8", 1201 "__sanitizer_cov_trace_switch", 1202 /* KMSAN */ 1203 "kmsan_copy_to_user", 1204 "kmsan_report", 1205 "kmsan_unpoison_entry_regs", 1206 "kmsan_unpoison_memory", 1207 "__msan_chain_origin", 1208 "__msan_get_context_state", 1209 "__msan_instrument_asm_store", 1210 "__msan_metadata_ptr_for_load_1", 1211 "__msan_metadata_ptr_for_load_2", 1212 "__msan_metadata_ptr_for_load_4", 1213 "__msan_metadata_ptr_for_load_8", 1214 "__msan_metadata_ptr_for_load_n", 1215 "__msan_metadata_ptr_for_store_1", 1216 "__msan_metadata_ptr_for_store_2", 1217 "__msan_metadata_ptr_for_store_4", 1218 "__msan_metadata_ptr_for_store_8", 1219 "__msan_metadata_ptr_for_store_n", 1220 "__msan_poison_alloca", 1221 "__msan_warning", 1222 /* UBSAN */ 1223 "ubsan_type_mismatch_common", 1224 "__ubsan_handle_type_mismatch", 1225 "__ubsan_handle_type_mismatch_v1", 1226 "__ubsan_handle_shift_out_of_bounds", 1227 /* misc */ 1228 "csum_partial_copy_generic", 1229 "copy_mc_fragile", 1230 "copy_mc_fragile_handle_tail", 1231 "copy_mc_enhanced_fast_string", 1232 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */ 1233 "clear_user_erms", 1234 "clear_user_rep_good", 1235 "clear_user_original", 1236 NULL 1237 }; 1238 1239 static void add_uaccess_safe(struct objtool_file *file) 1240 { 1241 struct symbol *func; 1242 const char **name; 1243 1244 if (!opts.uaccess) 1245 return; 1246 1247 for (name = uaccess_safe_builtin; *name; name++) { 1248 func = find_symbol_by_name(file->elf, *name); 1249 if (!func) 1250 continue; 1251 1252 func->uaccess_safe = true; 1253 } 1254 } 1255 1256 /* 1257 * FIXME: For now, just ignore any alternatives which add retpolines. This is 1258 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline. 1259 * But it at least allows objtool to understand the control flow *around* the 1260 * retpoline. 1261 */ 1262 static int add_ignore_alternatives(struct objtool_file *file) 1263 { 1264 struct section *sec; 1265 struct reloc *reloc; 1266 struct instruction *insn; 1267 1268 sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts"); 1269 if (!sec) 1270 return 0; 1271 1272 list_for_each_entry(reloc, &sec->reloc_list, list) { 1273 if (reloc->sym->type != STT_SECTION) { 1274 WARN("unexpected relocation symbol type in %s", sec->name); 1275 return -1; 1276 } 1277 1278 insn = find_insn(file, reloc->sym->sec, reloc->addend); 1279 if (!insn) { 1280 WARN("bad .discard.ignore_alts entry"); 1281 return -1; 1282 } 1283 1284 insn->ignore_alts = true; 1285 } 1286 1287 return 0; 1288 } 1289 1290 __weak bool arch_is_retpoline(struct symbol *sym) 1291 { 1292 return false; 1293 } 1294 1295 __weak bool arch_is_rethunk(struct symbol *sym) 1296 { 1297 return false; 1298 } 1299 1300 #define NEGATIVE_RELOC ((void *)-1L) 1301 1302 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) 1303 { 1304 if (insn->reloc == NEGATIVE_RELOC) 1305 return NULL; 1306 1307 if (!insn->reloc) { 1308 if (!file) 1309 return NULL; 1310 1311 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec, 1312 insn->offset, insn->len); 1313 if (!insn->reloc) { 1314 insn->reloc = NEGATIVE_RELOC; 1315 return NULL; 1316 } 1317 } 1318 1319 return insn->reloc; 1320 } 1321 1322 static void remove_insn_ops(struct instruction *insn) 1323 { 1324 struct stack_op *op, *tmp; 1325 1326 list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) { 1327 list_del(&op->list); 1328 free(op); 1329 } 1330 } 1331 1332 static void annotate_call_site(struct objtool_file *file, 1333 struct instruction *insn, bool sibling) 1334 { 1335 struct reloc *reloc = insn_reloc(file, insn); 1336 struct symbol *sym = insn->call_dest; 1337 1338 if (!sym) 1339 sym = reloc->sym; 1340 1341 /* 1342 * Alternative replacement code is just template code which is 1343 * sometimes copied to the original instruction. For now, don't 1344 * annotate it. (In the future we might consider annotating the 1345 * original instruction if/when it ever makes sense to do so.) 1346 */ 1347 if (!strcmp(insn->sec->name, ".altinstr_replacement")) 1348 return; 1349 1350 if (sym->static_call_tramp) { 1351 list_add_tail(&insn->call_node, &file->static_call_list); 1352 return; 1353 } 1354 1355 if (sym->retpoline_thunk) { 1356 list_add_tail(&insn->call_node, &file->retpoline_call_list); 1357 return; 1358 } 1359 1360 /* 1361 * Many compilers cannot disable KCOV or sanitizer calls with a function 1362 * attribute so they need a little help, NOP out any such calls from 1363 * noinstr text. 1364 */ 1365 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) { 1366 if (reloc) { 1367 reloc->type = R_NONE; 1368 elf_write_reloc(file->elf, reloc); 1369 } 1370 1371 elf_write_insn(file->elf, insn->sec, 1372 insn->offset, insn->len, 1373 sibling ? arch_ret_insn(insn->len) 1374 : arch_nop_insn(insn->len)); 1375 1376 insn->type = sibling ? INSN_RETURN : INSN_NOP; 1377 1378 if (sibling) { 1379 /* 1380 * We've replaced the tail-call JMP insn by two new 1381 * insn: RET; INT3, except we only have a single struct 1382 * insn here. Mark it retpoline_safe to avoid the SLS 1383 * warning, instead of adding another insn. 1384 */ 1385 insn->retpoline_safe = true; 1386 } 1387 1388 return; 1389 } 1390 1391 if (opts.mcount && sym->fentry) { 1392 if (sibling) 1393 WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset); 1394 if (opts.mnop) { 1395 if (reloc) { 1396 reloc->type = R_NONE; 1397 elf_write_reloc(file->elf, reloc); 1398 } 1399 1400 elf_write_insn(file->elf, insn->sec, 1401 insn->offset, insn->len, 1402 arch_nop_insn(insn->len)); 1403 1404 insn->type = INSN_NOP; 1405 } 1406 1407 list_add_tail(&insn->call_node, &file->mcount_loc_list); 1408 return; 1409 } 1410 1411 if (insn->type == INSN_CALL && !insn->sec->init) 1412 list_add_tail(&insn->call_node, &file->call_list); 1413 1414 if (!sibling && dead_end_function(file, sym)) 1415 insn->dead_end = true; 1416 } 1417 1418 static void add_call_dest(struct objtool_file *file, struct instruction *insn, 1419 struct symbol *dest, bool sibling) 1420 { 1421 insn->call_dest = dest; 1422 if (!dest) 1423 return; 1424 1425 /* 1426 * Whatever stack impact regular CALLs have, should be undone 1427 * by the RETURN of the called function. 1428 * 1429 * Annotated intra-function calls retain the stack_ops but 1430 * are converted to JUMP, see read_intra_function_calls(). 1431 */ 1432 remove_insn_ops(insn); 1433 1434 annotate_call_site(file, insn, sibling); 1435 } 1436 1437 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn) 1438 { 1439 /* 1440 * Retpoline calls/jumps are really dynamic calls/jumps in disguise, 1441 * so convert them accordingly. 1442 */ 1443 switch (insn->type) { 1444 case INSN_CALL: 1445 insn->type = INSN_CALL_DYNAMIC; 1446 break; 1447 case INSN_JUMP_UNCONDITIONAL: 1448 insn->type = INSN_JUMP_DYNAMIC; 1449 break; 1450 case INSN_JUMP_CONDITIONAL: 1451 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL; 1452 break; 1453 default: 1454 return; 1455 } 1456 1457 insn->retpoline_safe = true; 1458 1459 /* 1460 * Whatever stack impact regular CALLs have, should be undone 1461 * by the RETURN of the called function. 1462 * 1463 * Annotated intra-function calls retain the stack_ops but 1464 * are converted to JUMP, see read_intra_function_calls(). 1465 */ 1466 remove_insn_ops(insn); 1467 1468 annotate_call_site(file, insn, false); 1469 } 1470 1471 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) 1472 { 1473 /* 1474 * Return thunk tail calls are really just returns in disguise, 1475 * so convert them accordingly. 1476 */ 1477 insn->type = INSN_RETURN; 1478 insn->retpoline_safe = true; 1479 1480 if (add) 1481 list_add_tail(&insn->call_node, &file->return_thunk_list); 1482 } 1483 1484 static bool is_first_func_insn(struct objtool_file *file, 1485 struct instruction *insn, struct symbol *sym) 1486 { 1487 if (insn->offset == sym->offset) 1488 return true; 1489 1490 /* Allow direct CALL/JMP past ENDBR */ 1491 if (opts.ibt) { 1492 struct instruction *prev = prev_insn_same_sym(file, insn); 1493 1494 if (prev && prev->type == INSN_ENDBR && 1495 insn->offset == sym->offset + prev->len) 1496 return true; 1497 } 1498 1499 return false; 1500 } 1501 1502 /* 1503 * A sibling call is a tail-call to another symbol -- to differentiate from a 1504 * recursive tail-call which is to the same symbol. 1505 */ 1506 static bool jump_is_sibling_call(struct objtool_file *file, 1507 struct instruction *from, struct instruction *to) 1508 { 1509 struct symbol *fs = from->sym; 1510 struct symbol *ts = to->sym; 1511 1512 /* Not a sibling call if from/to a symbol hole */ 1513 if (!fs || !ts) 1514 return false; 1515 1516 /* Not a sibling call if not targeting the start of a symbol. */ 1517 if (!is_first_func_insn(file, to, ts)) 1518 return false; 1519 1520 /* Disallow sibling calls into STT_NOTYPE */ 1521 if (ts->type == STT_NOTYPE) 1522 return false; 1523 1524 /* Must not be self to be a sibling */ 1525 return fs->pfunc != ts->pfunc; 1526 } 1527 1528 /* 1529 * Find the destination instructions for all jumps. 1530 */ 1531 static int add_jump_destinations(struct objtool_file *file) 1532 { 1533 struct instruction *insn, *jump_dest; 1534 struct reloc *reloc; 1535 struct section *dest_sec; 1536 unsigned long dest_off; 1537 1538 for_each_insn(file, insn) { 1539 if (insn->jump_dest) { 1540 /* 1541 * handle_group_alt() may have previously set 1542 * 'jump_dest' for some alternatives. 1543 */ 1544 continue; 1545 } 1546 if (!is_static_jump(insn)) 1547 continue; 1548 1549 reloc = insn_reloc(file, insn); 1550 if (!reloc) { 1551 dest_sec = insn->sec; 1552 dest_off = arch_jump_destination(insn); 1553 } else if (reloc->sym->type == STT_SECTION) { 1554 dest_sec = reloc->sym->sec; 1555 dest_off = arch_dest_reloc_offset(reloc->addend); 1556 } else if (reloc->sym->retpoline_thunk) { 1557 add_retpoline_call(file, insn); 1558 continue; 1559 } else if (reloc->sym->return_thunk) { 1560 add_return_call(file, insn, true); 1561 continue; 1562 } else if (insn_func(insn)) { 1563 /* 1564 * External sibling call or internal sibling call with 1565 * STT_FUNC reloc. 1566 */ 1567 add_call_dest(file, insn, reloc->sym, true); 1568 continue; 1569 } else if (reloc->sym->sec->idx) { 1570 dest_sec = reloc->sym->sec; 1571 dest_off = reloc->sym->sym.st_value + 1572 arch_dest_reloc_offset(reloc->addend); 1573 } else { 1574 /* non-func asm code jumping to another file */ 1575 continue; 1576 } 1577 1578 jump_dest = find_insn(file, dest_sec, dest_off); 1579 if (!jump_dest) { 1580 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); 1581 1582 /* 1583 * This is a special case for zen_untrain_ret(). 1584 * It jumps to __x86_return_thunk(), but objtool 1585 * can't find the thunk's starting RET 1586 * instruction, because the RET is also in the 1587 * middle of another instruction. Objtool only 1588 * knows about the outer instruction. 1589 */ 1590 if (sym && sym->return_thunk) { 1591 add_return_call(file, insn, false); 1592 continue; 1593 } 1594 1595 WARN_FUNC("can't find jump dest instruction at %s+0x%lx", 1596 insn->sec, insn->offset, dest_sec->name, 1597 dest_off); 1598 return -1; 1599 } 1600 1601 /* 1602 * Cross-function jump. 1603 */ 1604 if (insn_func(insn) && insn_func(jump_dest) && 1605 insn_func(insn) != insn_func(jump_dest)) { 1606 1607 /* 1608 * For GCC 8+, create parent/child links for any cold 1609 * subfunctions. This is _mostly_ redundant with a 1610 * similar initialization in read_symbols(). 1611 * 1612 * If a function has aliases, we want the *first* such 1613 * function in the symbol table to be the subfunction's 1614 * parent. In that case we overwrite the 1615 * initialization done in read_symbols(). 1616 * 1617 * However this code can't completely replace the 1618 * read_symbols() code because this doesn't detect the 1619 * case where the parent function's only reference to a 1620 * subfunction is through a jump table. 1621 */ 1622 if (!strstr(insn_func(insn)->name, ".cold") && 1623 strstr(insn_func(jump_dest)->name, ".cold")) { 1624 insn_func(insn)->cfunc = insn_func(jump_dest); 1625 insn_func(jump_dest)->pfunc = insn_func(insn); 1626 } 1627 } 1628 1629 if (jump_is_sibling_call(file, insn, jump_dest)) { 1630 /* 1631 * Internal sibling call without reloc or with 1632 * STT_SECTION reloc. 1633 */ 1634 add_call_dest(file, insn, insn_func(jump_dest), true); 1635 continue; 1636 } 1637 1638 insn->jump_dest = jump_dest; 1639 } 1640 1641 return 0; 1642 } 1643 1644 static struct symbol *find_call_destination(struct section *sec, unsigned long offset) 1645 { 1646 struct symbol *call_dest; 1647 1648 call_dest = find_func_by_offset(sec, offset); 1649 if (!call_dest) 1650 call_dest = find_symbol_by_offset(sec, offset); 1651 1652 return call_dest; 1653 } 1654 1655 /* 1656 * Find the destination instructions for all calls. 1657 */ 1658 static int add_call_destinations(struct objtool_file *file) 1659 { 1660 struct instruction *insn; 1661 unsigned long dest_off; 1662 struct symbol *dest; 1663 struct reloc *reloc; 1664 1665 for_each_insn(file, insn) { 1666 if (insn->type != INSN_CALL) 1667 continue; 1668 1669 reloc = insn_reloc(file, insn); 1670 if (!reloc) { 1671 dest_off = arch_jump_destination(insn); 1672 dest = find_call_destination(insn->sec, dest_off); 1673 1674 add_call_dest(file, insn, dest, false); 1675 1676 if (insn->ignore) 1677 continue; 1678 1679 if (!insn->call_dest) { 1680 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset); 1681 return -1; 1682 } 1683 1684 if (insn_func(insn) && insn->call_dest->type != STT_FUNC) { 1685 WARN_FUNC("unsupported call to non-function", 1686 insn->sec, insn->offset); 1687 return -1; 1688 } 1689 1690 } else if (reloc->sym->type == STT_SECTION) { 1691 dest_off = arch_dest_reloc_offset(reloc->addend); 1692 dest = find_call_destination(reloc->sym->sec, dest_off); 1693 if (!dest) { 1694 WARN_FUNC("can't find call dest symbol at %s+0x%lx", 1695 insn->sec, insn->offset, 1696 reloc->sym->sec->name, 1697 dest_off); 1698 return -1; 1699 } 1700 1701 add_call_dest(file, insn, dest, false); 1702 1703 } else if (reloc->sym->retpoline_thunk) { 1704 add_retpoline_call(file, insn); 1705 1706 } else 1707 add_call_dest(file, insn, reloc->sym, false); 1708 } 1709 1710 return 0; 1711 } 1712 1713 /* 1714 * The .alternatives section requires some extra special care over and above 1715 * other special sections because alternatives are patched in place. 1716 */ 1717 static int handle_group_alt(struct objtool_file *file, 1718 struct special_alt *special_alt, 1719 struct instruction *orig_insn, 1720 struct instruction **new_insn) 1721 { 1722 struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL; 1723 struct alt_group *orig_alt_group, *new_alt_group; 1724 unsigned long dest_off; 1725 1726 1727 orig_alt_group = malloc(sizeof(*orig_alt_group)); 1728 if (!orig_alt_group) { 1729 WARN("malloc failed"); 1730 return -1; 1731 } 1732 orig_alt_group->cfi = calloc(special_alt->orig_len, 1733 sizeof(struct cfi_state *)); 1734 if (!orig_alt_group->cfi) { 1735 WARN("calloc failed"); 1736 return -1; 1737 } 1738 1739 last_orig_insn = NULL; 1740 insn = orig_insn; 1741 sec_for_each_insn_from(file, insn) { 1742 if (insn->offset >= special_alt->orig_off + special_alt->orig_len) 1743 break; 1744 1745 insn->alt_group = orig_alt_group; 1746 last_orig_insn = insn; 1747 } 1748 orig_alt_group->orig_group = NULL; 1749 orig_alt_group->first_insn = orig_insn; 1750 orig_alt_group->last_insn = last_orig_insn; 1751 1752 1753 new_alt_group = malloc(sizeof(*new_alt_group)); 1754 if (!new_alt_group) { 1755 WARN("malloc failed"); 1756 return -1; 1757 } 1758 1759 if (special_alt->new_len < special_alt->orig_len) { 1760 /* 1761 * Insert a fake nop at the end to make the replacement 1762 * alt_group the same size as the original. This is needed to 1763 * allow propagate_alt_cfi() to do its magic. When the last 1764 * instruction affects the stack, the instruction after it (the 1765 * nop) will propagate the new state to the shared CFI array. 1766 */ 1767 nop = malloc(sizeof(*nop)); 1768 if (!nop) { 1769 WARN("malloc failed"); 1770 return -1; 1771 } 1772 memset(nop, 0, sizeof(*nop)); 1773 INIT_LIST_HEAD(&nop->alts); 1774 INIT_LIST_HEAD(&nop->stack_ops); 1775 1776 nop->sec = special_alt->new_sec; 1777 nop->offset = special_alt->new_off + special_alt->new_len; 1778 nop->len = special_alt->orig_len - special_alt->new_len; 1779 nop->type = INSN_NOP; 1780 nop->sym = orig_insn->sym; 1781 nop->alt_group = new_alt_group; 1782 nop->ignore = orig_insn->ignore_alts; 1783 } 1784 1785 if (!special_alt->new_len) { 1786 *new_insn = nop; 1787 goto end; 1788 } 1789 1790 insn = *new_insn; 1791 sec_for_each_insn_from(file, insn) { 1792 struct reloc *alt_reloc; 1793 1794 if (insn->offset >= special_alt->new_off + special_alt->new_len) 1795 break; 1796 1797 last_new_insn = insn; 1798 1799 insn->ignore = orig_insn->ignore_alts; 1800 insn->sym = orig_insn->sym; 1801 insn->alt_group = new_alt_group; 1802 1803 /* 1804 * Since alternative replacement code is copy/pasted by the 1805 * kernel after applying relocations, generally such code can't 1806 * have relative-address relocation references to outside the 1807 * .altinstr_replacement section, unless the arch's 1808 * alternatives code can adjust the relative offsets 1809 * accordingly. 1810 */ 1811 alt_reloc = insn_reloc(file, insn); 1812 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) && 1813 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { 1814 1815 WARN_FUNC("unsupported relocation in alternatives section", 1816 insn->sec, insn->offset); 1817 return -1; 1818 } 1819 1820 if (!is_static_jump(insn)) 1821 continue; 1822 1823 if (!insn->immediate) 1824 continue; 1825 1826 dest_off = arch_jump_destination(insn); 1827 if (dest_off == special_alt->new_off + special_alt->new_len) { 1828 insn->jump_dest = next_insn_same_sec(file, last_orig_insn); 1829 if (!insn->jump_dest) { 1830 WARN_FUNC("can't find alternative jump destination", 1831 insn->sec, insn->offset); 1832 return -1; 1833 } 1834 } 1835 } 1836 1837 if (!last_new_insn) { 1838 WARN_FUNC("can't find last new alternative instruction", 1839 special_alt->new_sec, special_alt->new_off); 1840 return -1; 1841 } 1842 1843 if (nop) 1844 list_add(&nop->list, &last_new_insn->list); 1845 end: 1846 new_alt_group->orig_group = orig_alt_group; 1847 new_alt_group->first_insn = *new_insn; 1848 new_alt_group->last_insn = nop ? : last_new_insn; 1849 new_alt_group->cfi = orig_alt_group->cfi; 1850 return 0; 1851 } 1852 1853 /* 1854 * A jump table entry can either convert a nop to a jump or a jump to a nop. 1855 * If the original instruction is a jump, make the alt entry an effective nop 1856 * by just skipping the original instruction. 1857 */ 1858 static int handle_jump_alt(struct objtool_file *file, 1859 struct special_alt *special_alt, 1860 struct instruction *orig_insn, 1861 struct instruction **new_insn) 1862 { 1863 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && 1864 orig_insn->type != INSN_NOP) { 1865 1866 WARN_FUNC("unsupported instruction at jump label", 1867 orig_insn->sec, orig_insn->offset); 1868 return -1; 1869 } 1870 1871 if (opts.hack_jump_label && special_alt->key_addend & 2) { 1872 struct reloc *reloc = insn_reloc(file, orig_insn); 1873 1874 if (reloc) { 1875 reloc->type = R_NONE; 1876 elf_write_reloc(file->elf, reloc); 1877 } 1878 elf_write_insn(file->elf, orig_insn->sec, 1879 orig_insn->offset, orig_insn->len, 1880 arch_nop_insn(orig_insn->len)); 1881 orig_insn->type = INSN_NOP; 1882 } 1883 1884 if (orig_insn->type == INSN_NOP) { 1885 if (orig_insn->len == 2) 1886 file->jl_nop_short++; 1887 else 1888 file->jl_nop_long++; 1889 1890 return 0; 1891 } 1892 1893 if (orig_insn->len == 2) 1894 file->jl_short++; 1895 else 1896 file->jl_long++; 1897 1898 *new_insn = list_next_entry(orig_insn, list); 1899 return 0; 1900 } 1901 1902 /* 1903 * Read all the special sections which have alternate instructions which can be 1904 * patched in or redirected to at runtime. Each instruction having alternate 1905 * instruction(s) has them added to its insn->alts list, which will be 1906 * traversed in validate_branch(). 1907 */ 1908 static int add_special_section_alts(struct objtool_file *file) 1909 { 1910 struct list_head special_alts; 1911 struct instruction *orig_insn, *new_insn; 1912 struct special_alt *special_alt, *tmp; 1913 struct alternative *alt; 1914 int ret; 1915 1916 ret = special_get_alts(file->elf, &special_alts); 1917 if (ret) 1918 return ret; 1919 1920 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) { 1921 1922 orig_insn = find_insn(file, special_alt->orig_sec, 1923 special_alt->orig_off); 1924 if (!orig_insn) { 1925 WARN_FUNC("special: can't find orig instruction", 1926 special_alt->orig_sec, special_alt->orig_off); 1927 ret = -1; 1928 goto out; 1929 } 1930 1931 new_insn = NULL; 1932 if (!special_alt->group || special_alt->new_len) { 1933 new_insn = find_insn(file, special_alt->new_sec, 1934 special_alt->new_off); 1935 if (!new_insn) { 1936 WARN_FUNC("special: can't find new instruction", 1937 special_alt->new_sec, 1938 special_alt->new_off); 1939 ret = -1; 1940 goto out; 1941 } 1942 } 1943 1944 if (special_alt->group) { 1945 if (!special_alt->orig_len) { 1946 WARN_FUNC("empty alternative entry", 1947 orig_insn->sec, orig_insn->offset); 1948 continue; 1949 } 1950 1951 ret = handle_group_alt(file, special_alt, orig_insn, 1952 &new_insn); 1953 if (ret) 1954 goto out; 1955 } else if (special_alt->jump_or_nop) { 1956 ret = handle_jump_alt(file, special_alt, orig_insn, 1957 &new_insn); 1958 if (ret) 1959 goto out; 1960 } 1961 1962 alt = malloc(sizeof(*alt)); 1963 if (!alt) { 1964 WARN("malloc failed"); 1965 ret = -1; 1966 goto out; 1967 } 1968 1969 alt->insn = new_insn; 1970 alt->skip_orig = special_alt->skip_orig; 1971 orig_insn->ignore_alts |= special_alt->skip_alt; 1972 list_add_tail(&alt->list, &orig_insn->alts); 1973 1974 list_del(&special_alt->list); 1975 free(special_alt); 1976 } 1977 1978 if (opts.stats) { 1979 printf("jl\\\tNOP\tJMP\n"); 1980 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); 1981 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); 1982 } 1983 1984 out: 1985 return ret; 1986 } 1987 1988 static int add_jump_table(struct objtool_file *file, struct instruction *insn, 1989 struct reloc *table) 1990 { 1991 struct reloc *reloc = table; 1992 struct instruction *dest_insn; 1993 struct alternative *alt; 1994 struct symbol *pfunc = insn_func(insn)->pfunc; 1995 unsigned int prev_offset = 0; 1996 1997 /* 1998 * Each @reloc is a switch table relocation which points to the target 1999 * instruction. 2000 */ 2001 list_for_each_entry_from(reloc, &table->sec->reloc_list, list) { 2002 2003 /* Check for the end of the table: */ 2004 if (reloc != table && reloc->jump_table_start) 2005 break; 2006 2007 /* Make sure the table entries are consecutive: */ 2008 if (prev_offset && reloc->offset != prev_offset + 8) 2009 break; 2010 2011 /* Detect function pointers from contiguous objects: */ 2012 if (reloc->sym->sec == pfunc->sec && 2013 reloc->addend == pfunc->offset) 2014 break; 2015 2016 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend); 2017 if (!dest_insn) 2018 break; 2019 2020 /* Make sure the destination is in the same function: */ 2021 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc) 2022 break; 2023 2024 alt = malloc(sizeof(*alt)); 2025 if (!alt) { 2026 WARN("malloc failed"); 2027 return -1; 2028 } 2029 2030 alt->insn = dest_insn; 2031 list_add_tail(&alt->list, &insn->alts); 2032 prev_offset = reloc->offset; 2033 } 2034 2035 if (!prev_offset) { 2036 WARN_FUNC("can't find switch jump table", 2037 insn->sec, insn->offset); 2038 return -1; 2039 } 2040 2041 return 0; 2042 } 2043 2044 /* 2045 * find_jump_table() - Given a dynamic jump, find the switch jump table 2046 * associated with it. 2047 */ 2048 static struct reloc *find_jump_table(struct objtool_file *file, 2049 struct symbol *func, 2050 struct instruction *insn) 2051 { 2052 struct reloc *table_reloc; 2053 struct instruction *dest_insn, *orig_insn = insn; 2054 2055 /* 2056 * Backward search using the @first_jump_src links, these help avoid 2057 * much of the 'in between' code. Which avoids us getting confused by 2058 * it. 2059 */ 2060 for (; 2061 insn && insn_func(insn) && insn_func(insn)->pfunc == func; 2062 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) { 2063 2064 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC) 2065 break; 2066 2067 /* allow small jumps within the range */ 2068 if (insn->type == INSN_JUMP_UNCONDITIONAL && 2069 insn->jump_dest && 2070 (insn->jump_dest->offset <= insn->offset || 2071 insn->jump_dest->offset > orig_insn->offset)) 2072 break; 2073 2074 table_reloc = arch_find_switch_table(file, insn); 2075 if (!table_reloc) 2076 continue; 2077 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend); 2078 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func) 2079 continue; 2080 2081 return table_reloc; 2082 } 2083 2084 return NULL; 2085 } 2086 2087 /* 2088 * First pass: Mark the head of each jump table so that in the next pass, 2089 * we know when a given jump table ends and the next one starts. 2090 */ 2091 static void mark_func_jump_tables(struct objtool_file *file, 2092 struct symbol *func) 2093 { 2094 struct instruction *insn, *last = NULL; 2095 struct reloc *reloc; 2096 2097 func_for_each_insn(file, func, insn) { 2098 if (!last) 2099 last = insn; 2100 2101 /* 2102 * Store back-pointers for unconditional forward jumps such 2103 * that find_jump_table() can back-track using those and 2104 * avoid some potentially confusing code. 2105 */ 2106 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && 2107 insn->offset > last->offset && 2108 insn->jump_dest->offset > insn->offset && 2109 !insn->jump_dest->first_jump_src) { 2110 2111 insn->jump_dest->first_jump_src = insn; 2112 last = insn->jump_dest; 2113 } 2114 2115 if (insn->type != INSN_JUMP_DYNAMIC) 2116 continue; 2117 2118 reloc = find_jump_table(file, func, insn); 2119 if (reloc) { 2120 reloc->jump_table_start = true; 2121 insn->jump_table = reloc; 2122 } 2123 } 2124 } 2125 2126 static int add_func_jump_tables(struct objtool_file *file, 2127 struct symbol *func) 2128 { 2129 struct instruction *insn; 2130 int ret; 2131 2132 func_for_each_insn(file, func, insn) { 2133 if (!insn->jump_table) 2134 continue; 2135 2136 ret = add_jump_table(file, insn, insn->jump_table); 2137 if (ret) 2138 return ret; 2139 } 2140 2141 return 0; 2142 } 2143 2144 /* 2145 * For some switch statements, gcc generates a jump table in the .rodata 2146 * section which contains a list of addresses within the function to jump to. 2147 * This finds these jump tables and adds them to the insn->alts lists. 2148 */ 2149 static int add_jump_table_alts(struct objtool_file *file) 2150 { 2151 struct section *sec; 2152 struct symbol *func; 2153 int ret; 2154 2155 if (!file->rodata) 2156 return 0; 2157 2158 for_each_sec(file, sec) { 2159 list_for_each_entry(func, &sec->symbol_list, list) { 2160 if (func->type != STT_FUNC) 2161 continue; 2162 2163 mark_func_jump_tables(file, func); 2164 ret = add_func_jump_tables(file, func); 2165 if (ret) 2166 return ret; 2167 } 2168 } 2169 2170 return 0; 2171 } 2172 2173 static void set_func_state(struct cfi_state *state) 2174 { 2175 state->cfa = initial_func_cfi.cfa; 2176 memcpy(&state->regs, &initial_func_cfi.regs, 2177 CFI_NUM_REGS * sizeof(struct cfi_reg)); 2178 state->stack_size = initial_func_cfi.cfa.offset; 2179 } 2180 2181 static int read_unwind_hints(struct objtool_file *file) 2182 { 2183 struct cfi_state cfi = init_cfi; 2184 struct section *sec, *relocsec; 2185 struct unwind_hint *hint; 2186 struct instruction *insn; 2187 struct reloc *reloc; 2188 int i; 2189 2190 sec = find_section_by_name(file->elf, ".discard.unwind_hints"); 2191 if (!sec) 2192 return 0; 2193 2194 relocsec = sec->reloc; 2195 if (!relocsec) { 2196 WARN("missing .rela.discard.unwind_hints section"); 2197 return -1; 2198 } 2199 2200 if (sec->sh.sh_size % sizeof(struct unwind_hint)) { 2201 WARN("struct unwind_hint size mismatch"); 2202 return -1; 2203 } 2204 2205 file->hints = true; 2206 2207 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) { 2208 hint = (struct unwind_hint *)sec->data->d_buf + i; 2209 2210 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); 2211 if (!reloc) { 2212 WARN("can't find reloc for unwind_hints[%d]", i); 2213 return -1; 2214 } 2215 2216 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2217 if (!insn) { 2218 WARN("can't find insn for unwind_hints[%d]", i); 2219 return -1; 2220 } 2221 2222 insn->hint = true; 2223 2224 if (hint->type == UNWIND_HINT_TYPE_SAVE) { 2225 insn->hint = false; 2226 insn->save = true; 2227 continue; 2228 } 2229 2230 if (hint->type == UNWIND_HINT_TYPE_RESTORE) { 2231 insn->restore = true; 2232 continue; 2233 } 2234 2235 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 2236 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 2237 2238 if (sym && sym->bind == STB_GLOBAL) { 2239 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { 2240 WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR", 2241 insn->sec, insn->offset); 2242 } 2243 2244 insn->entry = 1; 2245 } 2246 } 2247 2248 if (hint->type == UNWIND_HINT_TYPE_ENTRY) { 2249 hint->type = UNWIND_HINT_TYPE_CALL; 2250 insn->entry = 1; 2251 } 2252 2253 if (hint->type == UNWIND_HINT_TYPE_FUNC) { 2254 insn->cfi = &func_cfi; 2255 continue; 2256 } 2257 2258 if (insn->cfi) 2259 cfi = *(insn->cfi); 2260 2261 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { 2262 WARN_FUNC("unsupported unwind_hint sp base reg %d", 2263 insn->sec, insn->offset, hint->sp_reg); 2264 return -1; 2265 } 2266 2267 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); 2268 cfi.type = hint->type; 2269 cfi.end = hint->end; 2270 2271 insn->cfi = cfi_hash_find_or_add(&cfi); 2272 } 2273 2274 return 0; 2275 } 2276 2277 static int read_noendbr_hints(struct objtool_file *file) 2278 { 2279 struct section *sec; 2280 struct instruction *insn; 2281 struct reloc *reloc; 2282 2283 sec = find_section_by_name(file->elf, ".rela.discard.noendbr"); 2284 if (!sec) 2285 return 0; 2286 2287 list_for_each_entry(reloc, &sec->reloc_list, list) { 2288 insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend); 2289 if (!insn) { 2290 WARN("bad .discard.noendbr entry"); 2291 return -1; 2292 } 2293 2294 insn->noendbr = 1; 2295 } 2296 2297 return 0; 2298 } 2299 2300 static int read_retpoline_hints(struct objtool_file *file) 2301 { 2302 struct section *sec; 2303 struct instruction *insn; 2304 struct reloc *reloc; 2305 2306 sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe"); 2307 if (!sec) 2308 return 0; 2309 2310 list_for_each_entry(reloc, &sec->reloc_list, list) { 2311 if (reloc->sym->type != STT_SECTION) { 2312 WARN("unexpected relocation symbol type in %s", sec->name); 2313 return -1; 2314 } 2315 2316 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2317 if (!insn) { 2318 WARN("bad .discard.retpoline_safe entry"); 2319 return -1; 2320 } 2321 2322 if (insn->type != INSN_JUMP_DYNAMIC && 2323 insn->type != INSN_CALL_DYNAMIC && 2324 insn->type != INSN_RETURN && 2325 insn->type != INSN_NOP) { 2326 WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop", 2327 insn->sec, insn->offset); 2328 return -1; 2329 } 2330 2331 insn->retpoline_safe = true; 2332 } 2333 2334 return 0; 2335 } 2336 2337 static int read_instr_hints(struct objtool_file *file) 2338 { 2339 struct section *sec; 2340 struct instruction *insn; 2341 struct reloc *reloc; 2342 2343 sec = find_section_by_name(file->elf, ".rela.discard.instr_end"); 2344 if (!sec) 2345 return 0; 2346 2347 list_for_each_entry(reloc, &sec->reloc_list, list) { 2348 if (reloc->sym->type != STT_SECTION) { 2349 WARN("unexpected relocation symbol type in %s", sec->name); 2350 return -1; 2351 } 2352 2353 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2354 if (!insn) { 2355 WARN("bad .discard.instr_end entry"); 2356 return -1; 2357 } 2358 2359 insn->instr--; 2360 } 2361 2362 sec = find_section_by_name(file->elf, ".rela.discard.instr_begin"); 2363 if (!sec) 2364 return 0; 2365 2366 list_for_each_entry(reloc, &sec->reloc_list, list) { 2367 if (reloc->sym->type != STT_SECTION) { 2368 WARN("unexpected relocation symbol type in %s", sec->name); 2369 return -1; 2370 } 2371 2372 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2373 if (!insn) { 2374 WARN("bad .discard.instr_begin entry"); 2375 return -1; 2376 } 2377 2378 insn->instr++; 2379 } 2380 2381 return 0; 2382 } 2383 2384 static int read_intra_function_calls(struct objtool_file *file) 2385 { 2386 struct instruction *insn; 2387 struct section *sec; 2388 struct reloc *reloc; 2389 2390 sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls"); 2391 if (!sec) 2392 return 0; 2393 2394 list_for_each_entry(reloc, &sec->reloc_list, list) { 2395 unsigned long dest_off; 2396 2397 if (reloc->sym->type != STT_SECTION) { 2398 WARN("unexpected relocation symbol type in %s", 2399 sec->name); 2400 return -1; 2401 } 2402 2403 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2404 if (!insn) { 2405 WARN("bad .discard.intra_function_call entry"); 2406 return -1; 2407 } 2408 2409 if (insn->type != INSN_CALL) { 2410 WARN_FUNC("intra_function_call not a direct call", 2411 insn->sec, insn->offset); 2412 return -1; 2413 } 2414 2415 /* 2416 * Treat intra-function CALLs as JMPs, but with a stack_op. 2417 * See add_call_destinations(), which strips stack_ops from 2418 * normal CALLs. 2419 */ 2420 insn->type = INSN_JUMP_UNCONDITIONAL; 2421 2422 dest_off = arch_jump_destination(insn); 2423 insn->jump_dest = find_insn(file, insn->sec, dest_off); 2424 if (!insn->jump_dest) { 2425 WARN_FUNC("can't find call dest at %s+0x%lx", 2426 insn->sec, insn->offset, 2427 insn->sec->name, dest_off); 2428 return -1; 2429 } 2430 } 2431 2432 return 0; 2433 } 2434 2435 /* 2436 * Return true if name matches an instrumentation function, where calls to that 2437 * function from noinstr code can safely be removed, but compilers won't do so. 2438 */ 2439 static bool is_profiling_func(const char *name) 2440 { 2441 /* 2442 * Many compilers cannot disable KCOV with a function attribute. 2443 */ 2444 if (!strncmp(name, "__sanitizer_cov_", 16)) 2445 return true; 2446 2447 /* 2448 * Some compilers currently do not remove __tsan_func_entry/exit nor 2449 * __tsan_atomic_signal_fence (used for barrier instrumentation) with 2450 * the __no_sanitize_thread attribute, remove them. Once the kernel's 2451 * minimum Clang version is 14.0, this can be removed. 2452 */ 2453 if (!strncmp(name, "__tsan_func_", 12) || 2454 !strcmp(name, "__tsan_atomic_signal_fence")) 2455 return true; 2456 2457 return false; 2458 } 2459 2460 static int classify_symbols(struct objtool_file *file) 2461 { 2462 struct section *sec; 2463 struct symbol *func; 2464 2465 for_each_sec(file, sec) { 2466 list_for_each_entry(func, &sec->symbol_list, list) { 2467 if (func->bind != STB_GLOBAL) 2468 continue; 2469 2470 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, 2471 strlen(STATIC_CALL_TRAMP_PREFIX_STR))) 2472 func->static_call_tramp = true; 2473 2474 if (arch_is_retpoline(func)) 2475 func->retpoline_thunk = true; 2476 2477 if (arch_is_rethunk(func)) 2478 func->return_thunk = true; 2479 2480 if (arch_ftrace_match(func->name)) 2481 func->fentry = true; 2482 2483 if (is_profiling_func(func->name)) 2484 func->profiling_func = true; 2485 } 2486 } 2487 2488 return 0; 2489 } 2490 2491 static void mark_rodata(struct objtool_file *file) 2492 { 2493 struct section *sec; 2494 bool found = false; 2495 2496 /* 2497 * Search for the following rodata sections, each of which can 2498 * potentially contain jump tables: 2499 * 2500 * - .rodata: can contain GCC switch tables 2501 * - .rodata.<func>: same, if -fdata-sections is being used 2502 * - .rodata..c_jump_table: contains C annotated jump tables 2503 * 2504 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2505 */ 2506 for_each_sec(file, sec) { 2507 if (!strncmp(sec->name, ".rodata", 7) && 2508 !strstr(sec->name, ".str1.")) { 2509 sec->rodata = true; 2510 found = true; 2511 } 2512 } 2513 2514 file->rodata = found; 2515 } 2516 2517 static int decode_sections(struct objtool_file *file) 2518 { 2519 int ret; 2520 2521 mark_rodata(file); 2522 2523 ret = init_pv_ops(file); 2524 if (ret) 2525 return ret; 2526 2527 /* 2528 * Must be before add_{jump_call}_destination. 2529 */ 2530 ret = classify_symbols(file); 2531 if (ret) 2532 return ret; 2533 2534 ret = decode_instructions(file); 2535 if (ret) 2536 return ret; 2537 2538 add_ignores(file); 2539 add_uaccess_safe(file); 2540 2541 ret = add_ignore_alternatives(file); 2542 if (ret) 2543 return ret; 2544 2545 /* 2546 * Must be before read_unwind_hints() since that needs insn->noendbr. 2547 */ 2548 ret = read_noendbr_hints(file); 2549 if (ret) 2550 return ret; 2551 2552 /* 2553 * Must be before add_jump_destinations(), which depends on 'func' 2554 * being set for alternatives, to enable proper sibling call detection. 2555 */ 2556 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) { 2557 ret = add_special_section_alts(file); 2558 if (ret) 2559 return ret; 2560 } 2561 2562 ret = add_jump_destinations(file); 2563 if (ret) 2564 return ret; 2565 2566 /* 2567 * Must be before add_call_destination(); it changes INSN_CALL to 2568 * INSN_JUMP. 2569 */ 2570 ret = read_intra_function_calls(file); 2571 if (ret) 2572 return ret; 2573 2574 ret = add_call_destinations(file); 2575 if (ret) 2576 return ret; 2577 2578 /* 2579 * Must be after add_call_destinations() such that it can override 2580 * dead_end_function() marks. 2581 */ 2582 ret = add_dead_ends(file); 2583 if (ret) 2584 return ret; 2585 2586 ret = add_jump_table_alts(file); 2587 if (ret) 2588 return ret; 2589 2590 ret = read_unwind_hints(file); 2591 if (ret) 2592 return ret; 2593 2594 ret = read_retpoline_hints(file); 2595 if (ret) 2596 return ret; 2597 2598 ret = read_instr_hints(file); 2599 if (ret) 2600 return ret; 2601 2602 return 0; 2603 } 2604 2605 static bool is_fentry_call(struct instruction *insn) 2606 { 2607 if (insn->type == INSN_CALL && 2608 insn->call_dest && 2609 insn->call_dest->fentry) 2610 return true; 2611 2612 return false; 2613 } 2614 2615 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state) 2616 { 2617 struct cfi_state *cfi = &state->cfi; 2618 int i; 2619 2620 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap) 2621 return true; 2622 2623 if (cfi->cfa.offset != initial_func_cfi.cfa.offset) 2624 return true; 2625 2626 if (cfi->stack_size != initial_func_cfi.cfa.offset) 2627 return true; 2628 2629 for (i = 0; i < CFI_NUM_REGS; i++) { 2630 if (cfi->regs[i].base != initial_func_cfi.regs[i].base || 2631 cfi->regs[i].offset != initial_func_cfi.regs[i].offset) 2632 return true; 2633 } 2634 2635 return false; 2636 } 2637 2638 static bool check_reg_frame_pos(const struct cfi_reg *reg, 2639 int expected_offset) 2640 { 2641 return reg->base == CFI_CFA && 2642 reg->offset == expected_offset; 2643 } 2644 2645 static bool has_valid_stack_frame(struct insn_state *state) 2646 { 2647 struct cfi_state *cfi = &state->cfi; 2648 2649 if (cfi->cfa.base == CFI_BP && 2650 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) && 2651 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8)) 2652 return true; 2653 2654 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP) 2655 return true; 2656 2657 return false; 2658 } 2659 2660 static int update_cfi_state_regs(struct instruction *insn, 2661 struct cfi_state *cfi, 2662 struct stack_op *op) 2663 { 2664 struct cfi_reg *cfa = &cfi->cfa; 2665 2666 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT) 2667 return 0; 2668 2669 /* push */ 2670 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF) 2671 cfa->offset += 8; 2672 2673 /* pop */ 2674 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF) 2675 cfa->offset -= 8; 2676 2677 /* add immediate to sp */ 2678 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD && 2679 op->dest.reg == CFI_SP && op->src.reg == CFI_SP) 2680 cfa->offset -= op->src.offset; 2681 2682 return 0; 2683 } 2684 2685 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset) 2686 { 2687 if (arch_callee_saved_reg(reg) && 2688 cfi->regs[reg].base == CFI_UNDEFINED) { 2689 cfi->regs[reg].base = base; 2690 cfi->regs[reg].offset = offset; 2691 } 2692 } 2693 2694 static void restore_reg(struct cfi_state *cfi, unsigned char reg) 2695 { 2696 cfi->regs[reg].base = initial_func_cfi.regs[reg].base; 2697 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset; 2698 } 2699 2700 /* 2701 * A note about DRAP stack alignment: 2702 * 2703 * GCC has the concept of a DRAP register, which is used to help keep track of 2704 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP 2705 * register. The typical DRAP pattern is: 2706 * 2707 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10 2708 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp 2709 * 41 ff 72 f8 pushq -0x8(%r10) 2710 * 55 push %rbp 2711 * 48 89 e5 mov %rsp,%rbp 2712 * (more pushes) 2713 * 41 52 push %r10 2714 * ... 2715 * 41 5a pop %r10 2716 * (more pops) 2717 * 5d pop %rbp 2718 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2719 * c3 retq 2720 * 2721 * There are some variations in the epilogues, like: 2722 * 2723 * 5b pop %rbx 2724 * 41 5a pop %r10 2725 * 41 5c pop %r12 2726 * 41 5d pop %r13 2727 * 41 5e pop %r14 2728 * c9 leaveq 2729 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2730 * c3 retq 2731 * 2732 * and: 2733 * 2734 * 4c 8b 55 e8 mov -0x18(%rbp),%r10 2735 * 48 8b 5d e0 mov -0x20(%rbp),%rbx 2736 * 4c 8b 65 f0 mov -0x10(%rbp),%r12 2737 * 4c 8b 6d f8 mov -0x8(%rbp),%r13 2738 * c9 leaveq 2739 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2740 * c3 retq 2741 * 2742 * Sometimes r13 is used as the DRAP register, in which case it's saved and 2743 * restored beforehand: 2744 * 2745 * 41 55 push %r13 2746 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13 2747 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp 2748 * ... 2749 * 49 8d 65 f0 lea -0x10(%r13),%rsp 2750 * 41 5d pop %r13 2751 * c3 retq 2752 */ 2753 static int update_cfi_state(struct instruction *insn, 2754 struct instruction *next_insn, 2755 struct cfi_state *cfi, struct stack_op *op) 2756 { 2757 struct cfi_reg *cfa = &cfi->cfa; 2758 struct cfi_reg *regs = cfi->regs; 2759 2760 /* stack operations don't make sense with an undefined CFA */ 2761 if (cfa->base == CFI_UNDEFINED) { 2762 if (insn_func(insn)) { 2763 WARN_FUNC("undefined stack state", insn->sec, insn->offset); 2764 return -1; 2765 } 2766 return 0; 2767 } 2768 2769 if (cfi->type == UNWIND_HINT_TYPE_REGS || 2770 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) 2771 return update_cfi_state_regs(insn, cfi, op); 2772 2773 switch (op->dest.type) { 2774 2775 case OP_DEST_REG: 2776 switch (op->src.type) { 2777 2778 case OP_SRC_REG: 2779 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP && 2780 cfa->base == CFI_SP && 2781 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) { 2782 2783 /* mov %rsp, %rbp */ 2784 cfa->base = op->dest.reg; 2785 cfi->bp_scratch = false; 2786 } 2787 2788 else if (op->src.reg == CFI_SP && 2789 op->dest.reg == CFI_BP && cfi->drap) { 2790 2791 /* drap: mov %rsp, %rbp */ 2792 regs[CFI_BP].base = CFI_BP; 2793 regs[CFI_BP].offset = -cfi->stack_size; 2794 cfi->bp_scratch = false; 2795 } 2796 2797 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2798 2799 /* 2800 * mov %rsp, %reg 2801 * 2802 * This is needed for the rare case where GCC 2803 * does: 2804 * 2805 * mov %rsp, %rax 2806 * ... 2807 * mov %rax, %rsp 2808 */ 2809 cfi->vals[op->dest.reg].base = CFI_CFA; 2810 cfi->vals[op->dest.reg].offset = -cfi->stack_size; 2811 } 2812 2813 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP && 2814 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) { 2815 2816 /* 2817 * mov %rbp, %rsp 2818 * 2819 * Restore the original stack pointer (Clang). 2820 */ 2821 cfi->stack_size = -cfi->regs[CFI_BP].offset; 2822 } 2823 2824 else if (op->dest.reg == cfa->base) { 2825 2826 /* mov %reg, %rsp */ 2827 if (cfa->base == CFI_SP && 2828 cfi->vals[op->src.reg].base == CFI_CFA) { 2829 2830 /* 2831 * This is needed for the rare case 2832 * where GCC does something dumb like: 2833 * 2834 * lea 0x8(%rsp), %rcx 2835 * ... 2836 * mov %rcx, %rsp 2837 */ 2838 cfa->offset = -cfi->vals[op->src.reg].offset; 2839 cfi->stack_size = cfa->offset; 2840 2841 } else if (cfa->base == CFI_SP && 2842 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2843 cfi->vals[op->src.reg].offset == cfa->offset) { 2844 2845 /* 2846 * Stack swizzle: 2847 * 2848 * 1: mov %rsp, (%[tos]) 2849 * 2: mov %[tos], %rsp 2850 * ... 2851 * 3: pop %rsp 2852 * 2853 * Where: 2854 * 2855 * 1 - places a pointer to the previous 2856 * stack at the Top-of-Stack of the 2857 * new stack. 2858 * 2859 * 2 - switches to the new stack. 2860 * 2861 * 3 - pops the Top-of-Stack to restore 2862 * the original stack. 2863 * 2864 * Note: we set base to SP_INDIRECT 2865 * here and preserve offset. Therefore 2866 * when the unwinder reaches ToS it 2867 * will dereference SP and then add the 2868 * offset to find the next frame, IOW: 2869 * (%rsp) + offset. 2870 */ 2871 cfa->base = CFI_SP_INDIRECT; 2872 2873 } else { 2874 cfa->base = CFI_UNDEFINED; 2875 cfa->offset = 0; 2876 } 2877 } 2878 2879 else if (op->dest.reg == CFI_SP && 2880 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2881 cfi->vals[op->src.reg].offset == cfa->offset) { 2882 2883 /* 2884 * The same stack swizzle case 2) as above. But 2885 * because we can't change cfa->base, case 3) 2886 * will become a regular POP. Pretend we're a 2887 * PUSH so things don't go unbalanced. 2888 */ 2889 cfi->stack_size += 8; 2890 } 2891 2892 2893 break; 2894 2895 case OP_SRC_ADD: 2896 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) { 2897 2898 /* add imm, %rsp */ 2899 cfi->stack_size -= op->src.offset; 2900 if (cfa->base == CFI_SP) 2901 cfa->offset -= op->src.offset; 2902 break; 2903 } 2904 2905 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { 2906 2907 /* lea disp(%rbp), %rsp */ 2908 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); 2909 break; 2910 } 2911 2912 if (!cfi->drap && op->src.reg == CFI_SP && 2913 op->dest.reg == CFI_BP && cfa->base == CFI_SP && 2914 check_reg_frame_pos(®s[CFI_BP], -cfa->offset + op->src.offset)) { 2915 2916 /* lea disp(%rsp), %rbp */ 2917 cfa->base = CFI_BP; 2918 cfa->offset -= op->src.offset; 2919 cfi->bp_scratch = false; 2920 break; 2921 } 2922 2923 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2924 2925 /* drap: lea disp(%rsp), %drap */ 2926 cfi->drap_reg = op->dest.reg; 2927 2928 /* 2929 * lea disp(%rsp), %reg 2930 * 2931 * This is needed for the rare case where GCC 2932 * does something dumb like: 2933 * 2934 * lea 0x8(%rsp), %rcx 2935 * ... 2936 * mov %rcx, %rsp 2937 */ 2938 cfi->vals[op->dest.reg].base = CFI_CFA; 2939 cfi->vals[op->dest.reg].offset = \ 2940 -cfi->stack_size + op->src.offset; 2941 2942 break; 2943 } 2944 2945 if (cfi->drap && op->dest.reg == CFI_SP && 2946 op->src.reg == cfi->drap_reg) { 2947 2948 /* drap: lea disp(%drap), %rsp */ 2949 cfa->base = CFI_SP; 2950 cfa->offset = cfi->stack_size = -op->src.offset; 2951 cfi->drap_reg = CFI_UNDEFINED; 2952 cfi->drap = false; 2953 break; 2954 } 2955 2956 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { 2957 WARN_FUNC("unsupported stack register modification", 2958 insn->sec, insn->offset); 2959 return -1; 2960 } 2961 2962 break; 2963 2964 case OP_SRC_AND: 2965 if (op->dest.reg != CFI_SP || 2966 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || 2967 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { 2968 WARN_FUNC("unsupported stack pointer realignment", 2969 insn->sec, insn->offset); 2970 return -1; 2971 } 2972 2973 if (cfi->drap_reg != CFI_UNDEFINED) { 2974 /* drap: and imm, %rsp */ 2975 cfa->base = cfi->drap_reg; 2976 cfa->offset = cfi->stack_size = 0; 2977 cfi->drap = true; 2978 } 2979 2980 /* 2981 * Older versions of GCC (4.8ish) realign the stack 2982 * without DRAP, with a frame pointer. 2983 */ 2984 2985 break; 2986 2987 case OP_SRC_POP: 2988 case OP_SRC_POPF: 2989 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) { 2990 2991 /* pop %rsp; # restore from a stack swizzle */ 2992 cfa->base = CFI_SP; 2993 break; 2994 } 2995 2996 if (!cfi->drap && op->dest.reg == cfa->base) { 2997 2998 /* pop %rbp */ 2999 cfa->base = CFI_SP; 3000 } 3001 3002 if (cfi->drap && cfa->base == CFI_BP_INDIRECT && 3003 op->dest.reg == cfi->drap_reg && 3004 cfi->drap_offset == -cfi->stack_size) { 3005 3006 /* drap: pop %drap */ 3007 cfa->base = cfi->drap_reg; 3008 cfa->offset = 0; 3009 cfi->drap_offset = -1; 3010 3011 } else if (cfi->stack_size == -regs[op->dest.reg].offset) { 3012 3013 /* pop %reg */ 3014 restore_reg(cfi, op->dest.reg); 3015 } 3016 3017 cfi->stack_size -= 8; 3018 if (cfa->base == CFI_SP) 3019 cfa->offset -= 8; 3020 3021 break; 3022 3023 case OP_SRC_REG_INDIRECT: 3024 if (!cfi->drap && op->dest.reg == cfa->base && 3025 op->dest.reg == CFI_BP) { 3026 3027 /* mov disp(%rsp), %rbp */ 3028 cfa->base = CFI_SP; 3029 cfa->offset = cfi->stack_size; 3030 } 3031 3032 if (cfi->drap && op->src.reg == CFI_BP && 3033 op->src.offset == cfi->drap_offset) { 3034 3035 /* drap: mov disp(%rbp), %drap */ 3036 cfa->base = cfi->drap_reg; 3037 cfa->offset = 0; 3038 cfi->drap_offset = -1; 3039 } 3040 3041 if (cfi->drap && op->src.reg == CFI_BP && 3042 op->src.offset == regs[op->dest.reg].offset) { 3043 3044 /* drap: mov disp(%rbp), %reg */ 3045 restore_reg(cfi, op->dest.reg); 3046 3047 } else if (op->src.reg == cfa->base && 3048 op->src.offset == regs[op->dest.reg].offset + cfa->offset) { 3049 3050 /* mov disp(%rbp), %reg */ 3051 /* mov disp(%rsp), %reg */ 3052 restore_reg(cfi, op->dest.reg); 3053 3054 } else if (op->src.reg == CFI_SP && 3055 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) { 3056 3057 /* mov disp(%rsp), %reg */ 3058 restore_reg(cfi, op->dest.reg); 3059 } 3060 3061 break; 3062 3063 default: 3064 WARN_FUNC("unknown stack-related instruction", 3065 insn->sec, insn->offset); 3066 return -1; 3067 } 3068 3069 break; 3070 3071 case OP_DEST_PUSH: 3072 case OP_DEST_PUSHF: 3073 cfi->stack_size += 8; 3074 if (cfa->base == CFI_SP) 3075 cfa->offset += 8; 3076 3077 if (op->src.type != OP_SRC_REG) 3078 break; 3079 3080 if (cfi->drap) { 3081 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3082 3083 /* drap: push %drap */ 3084 cfa->base = CFI_BP_INDIRECT; 3085 cfa->offset = -cfi->stack_size; 3086 3087 /* save drap so we know when to restore it */ 3088 cfi->drap_offset = -cfi->stack_size; 3089 3090 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) { 3091 3092 /* drap: push %rbp */ 3093 cfi->stack_size = 0; 3094 3095 } else { 3096 3097 /* drap: push %reg */ 3098 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); 3099 } 3100 3101 } else { 3102 3103 /* push %reg */ 3104 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size); 3105 } 3106 3107 /* detect when asm code uses rbp as a scratch register */ 3108 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP && 3109 cfa->base != CFI_BP) 3110 cfi->bp_scratch = true; 3111 break; 3112 3113 case OP_DEST_REG_INDIRECT: 3114 3115 if (cfi->drap) { 3116 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3117 3118 /* drap: mov %drap, disp(%rbp) */ 3119 cfa->base = CFI_BP_INDIRECT; 3120 cfa->offset = op->dest.offset; 3121 3122 /* save drap offset so we know when to restore it */ 3123 cfi->drap_offset = op->dest.offset; 3124 } else { 3125 3126 /* drap: mov reg, disp(%rbp) */ 3127 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); 3128 } 3129 3130 } else if (op->dest.reg == cfa->base) { 3131 3132 /* mov reg, disp(%rbp) */ 3133 /* mov reg, disp(%rsp) */ 3134 save_reg(cfi, op->src.reg, CFI_CFA, 3135 op->dest.offset - cfi->cfa.offset); 3136 3137 } else if (op->dest.reg == CFI_SP) { 3138 3139 /* mov reg, disp(%rsp) */ 3140 save_reg(cfi, op->src.reg, CFI_CFA, 3141 op->dest.offset - cfi->stack_size); 3142 3143 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) { 3144 3145 /* mov %rsp, (%reg); # setup a stack swizzle. */ 3146 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT; 3147 cfi->vals[op->dest.reg].offset = cfa->offset; 3148 } 3149 3150 break; 3151 3152 case OP_DEST_MEM: 3153 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { 3154 WARN_FUNC("unknown stack-related memory operation", 3155 insn->sec, insn->offset); 3156 return -1; 3157 } 3158 3159 /* pop mem */ 3160 cfi->stack_size -= 8; 3161 if (cfa->base == CFI_SP) 3162 cfa->offset -= 8; 3163 3164 break; 3165 3166 default: 3167 WARN_FUNC("unknown stack-related instruction", 3168 insn->sec, insn->offset); 3169 return -1; 3170 } 3171 3172 return 0; 3173 } 3174 3175 /* 3176 * The stack layouts of alternatives instructions can sometimes diverge when 3177 * they have stack modifications. That's fine as long as the potential stack 3178 * layouts don't conflict at any given potential instruction boundary. 3179 * 3180 * Flatten the CFIs of the different alternative code streams (both original 3181 * and replacement) into a single shared CFI array which can be used to detect 3182 * conflicts and nicely feed a linear array of ORC entries to the unwinder. 3183 */ 3184 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn) 3185 { 3186 struct cfi_state **alt_cfi; 3187 int group_off; 3188 3189 if (!insn->alt_group) 3190 return 0; 3191 3192 if (!insn->cfi) { 3193 WARN("CFI missing"); 3194 return -1; 3195 } 3196 3197 alt_cfi = insn->alt_group->cfi; 3198 group_off = insn->offset - insn->alt_group->first_insn->offset; 3199 3200 if (!alt_cfi[group_off]) { 3201 alt_cfi[group_off] = insn->cfi; 3202 } else { 3203 if (cficmp(alt_cfi[group_off], insn->cfi)) { 3204 WARN_FUNC("stack layout conflict in alternatives", 3205 insn->sec, insn->offset); 3206 return -1; 3207 } 3208 } 3209 3210 return 0; 3211 } 3212 3213 static int handle_insn_ops(struct instruction *insn, 3214 struct instruction *next_insn, 3215 struct insn_state *state) 3216 { 3217 struct stack_op *op; 3218 3219 list_for_each_entry(op, &insn->stack_ops, list) { 3220 3221 if (update_cfi_state(insn, next_insn, &state->cfi, op)) 3222 return 1; 3223 3224 if (!insn->alt_group) 3225 continue; 3226 3227 if (op->dest.type == OP_DEST_PUSHF) { 3228 if (!state->uaccess_stack) { 3229 state->uaccess_stack = 1; 3230 } else if (state->uaccess_stack >> 31) { 3231 WARN_FUNC("PUSHF stack exhausted", 3232 insn->sec, insn->offset); 3233 return 1; 3234 } 3235 state->uaccess_stack <<= 1; 3236 state->uaccess_stack |= state->uaccess; 3237 } 3238 3239 if (op->src.type == OP_SRC_POPF) { 3240 if (state->uaccess_stack) { 3241 state->uaccess = state->uaccess_stack & 1; 3242 state->uaccess_stack >>= 1; 3243 if (state->uaccess_stack == 1) 3244 state->uaccess_stack = 0; 3245 } 3246 } 3247 } 3248 3249 return 0; 3250 } 3251 3252 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) 3253 { 3254 struct cfi_state *cfi1 = insn->cfi; 3255 int i; 3256 3257 if (!cfi1) { 3258 WARN("CFI missing"); 3259 return false; 3260 } 3261 3262 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { 3263 3264 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d", 3265 insn->sec, insn->offset, 3266 cfi1->cfa.base, cfi1->cfa.offset, 3267 cfi2->cfa.base, cfi2->cfa.offset); 3268 3269 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) { 3270 for (i = 0; i < CFI_NUM_REGS; i++) { 3271 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], 3272 sizeof(struct cfi_reg))) 3273 continue; 3274 3275 WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", 3276 insn->sec, insn->offset, 3277 i, cfi1->regs[i].base, cfi1->regs[i].offset, 3278 i, cfi2->regs[i].base, cfi2->regs[i].offset); 3279 break; 3280 } 3281 3282 } else if (cfi1->type != cfi2->type) { 3283 3284 WARN_FUNC("stack state mismatch: type1=%d type2=%d", 3285 insn->sec, insn->offset, cfi1->type, cfi2->type); 3286 3287 } else if (cfi1->drap != cfi2->drap || 3288 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || 3289 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { 3290 3291 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", 3292 insn->sec, insn->offset, 3293 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, 3294 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); 3295 3296 } else 3297 return true; 3298 3299 return false; 3300 } 3301 3302 static inline bool func_uaccess_safe(struct symbol *func) 3303 { 3304 if (func) 3305 return func->uaccess_safe; 3306 3307 return false; 3308 } 3309 3310 static inline const char *call_dest_name(struct instruction *insn) 3311 { 3312 static char pvname[19]; 3313 struct reloc *rel; 3314 int idx; 3315 3316 if (insn->call_dest) 3317 return insn->call_dest->name; 3318 3319 rel = insn_reloc(NULL, insn); 3320 if (rel && !strcmp(rel->sym->name, "pv_ops")) { 3321 idx = (rel->addend / sizeof(void *)); 3322 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx); 3323 return pvname; 3324 } 3325 3326 return "{dynamic}"; 3327 } 3328 3329 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn) 3330 { 3331 struct symbol *target; 3332 struct reloc *rel; 3333 int idx; 3334 3335 rel = insn_reloc(file, insn); 3336 if (!rel || strcmp(rel->sym->name, "pv_ops")) 3337 return false; 3338 3339 idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *)); 3340 3341 if (file->pv_ops[idx].clean) 3342 return true; 3343 3344 file->pv_ops[idx].clean = true; 3345 3346 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) { 3347 if (!target->sec->noinstr) { 3348 WARN("pv_ops[%d]: %s", idx, target->name); 3349 file->pv_ops[idx].clean = false; 3350 } 3351 } 3352 3353 return file->pv_ops[idx].clean; 3354 } 3355 3356 static inline bool noinstr_call_dest(struct objtool_file *file, 3357 struct instruction *insn, 3358 struct symbol *func) 3359 { 3360 /* 3361 * We can't deal with indirect function calls at present; 3362 * assume they're instrumented. 3363 */ 3364 if (!func) { 3365 if (file->pv_ops) 3366 return pv_call_dest(file, insn); 3367 3368 return false; 3369 } 3370 3371 /* 3372 * If the symbol is from a noinstr section; we good. 3373 */ 3374 if (func->sec->noinstr) 3375 return true; 3376 3377 /* 3378 * The __ubsan_handle_*() calls are like WARN(), they only happen when 3379 * something 'BAD' happened. At the risk of taking the machine down, 3380 * let them proceed to get the message out. 3381 */ 3382 if (!strncmp(func->name, "__ubsan_handle_", 15)) 3383 return true; 3384 3385 return false; 3386 } 3387 3388 static int validate_call(struct objtool_file *file, 3389 struct instruction *insn, 3390 struct insn_state *state) 3391 { 3392 if (state->noinstr && state->instr <= 0 && 3393 !noinstr_call_dest(file, insn, insn->call_dest)) { 3394 WARN_FUNC("call to %s() leaves .noinstr.text section", 3395 insn->sec, insn->offset, call_dest_name(insn)); 3396 return 1; 3397 } 3398 3399 if (state->uaccess && !func_uaccess_safe(insn->call_dest)) { 3400 WARN_FUNC("call to %s() with UACCESS enabled", 3401 insn->sec, insn->offset, call_dest_name(insn)); 3402 return 1; 3403 } 3404 3405 if (state->df) { 3406 WARN_FUNC("call to %s() with DF set", 3407 insn->sec, insn->offset, call_dest_name(insn)); 3408 return 1; 3409 } 3410 3411 return 0; 3412 } 3413 3414 static int validate_sibling_call(struct objtool_file *file, 3415 struct instruction *insn, 3416 struct insn_state *state) 3417 { 3418 if (insn_func(insn) && has_modified_stack_frame(insn, state)) { 3419 WARN_FUNC("sibling call from callable instruction with modified stack frame", 3420 insn->sec, insn->offset); 3421 return 1; 3422 } 3423 3424 return validate_call(file, insn, state); 3425 } 3426 3427 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 3428 { 3429 if (state->noinstr && state->instr > 0) { 3430 WARN_FUNC("return with instrumentation enabled", 3431 insn->sec, insn->offset); 3432 return 1; 3433 } 3434 3435 if (state->uaccess && !func_uaccess_safe(func)) { 3436 WARN_FUNC("return with UACCESS enabled", 3437 insn->sec, insn->offset); 3438 return 1; 3439 } 3440 3441 if (!state->uaccess && func_uaccess_safe(func)) { 3442 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function", 3443 insn->sec, insn->offset); 3444 return 1; 3445 } 3446 3447 if (state->df) { 3448 WARN_FUNC("return with DF set", 3449 insn->sec, insn->offset); 3450 return 1; 3451 } 3452 3453 if (func && has_modified_stack_frame(insn, state)) { 3454 WARN_FUNC("return with modified stack frame", 3455 insn->sec, insn->offset); 3456 return 1; 3457 } 3458 3459 if (state->cfi.bp_scratch) { 3460 WARN_FUNC("BP used as a scratch register", 3461 insn->sec, insn->offset); 3462 return 1; 3463 } 3464 3465 return 0; 3466 } 3467 3468 static struct instruction *next_insn_to_validate(struct objtool_file *file, 3469 struct instruction *insn) 3470 { 3471 struct alt_group *alt_group = insn->alt_group; 3472 3473 /* 3474 * Simulate the fact that alternatives are patched in-place. When the 3475 * end of a replacement alt_group is reached, redirect objtool flow to 3476 * the end of the original alt_group. 3477 */ 3478 if (alt_group && insn == alt_group->last_insn && alt_group->orig_group) 3479 return next_insn_same_sec(file, alt_group->orig_group->last_insn); 3480 3481 return next_insn_same_sec(file, insn); 3482 } 3483 3484 /* 3485 * Follow the branch starting at the given instruction, and recursively follow 3486 * any other branches (jumps). Meanwhile, track the frame pointer state at 3487 * each instruction and validate all the rules described in 3488 * tools/objtool/Documentation/objtool.txt. 3489 */ 3490 static int validate_branch(struct objtool_file *file, struct symbol *func, 3491 struct instruction *insn, struct insn_state state) 3492 { 3493 struct alternative *alt; 3494 struct instruction *next_insn, *prev_insn = NULL; 3495 struct section *sec; 3496 u8 visited; 3497 int ret; 3498 3499 sec = insn->sec; 3500 3501 while (1) { 3502 next_insn = next_insn_to_validate(file, insn); 3503 3504 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { 3505 /* Ignore KCFI type preambles, which always fall through */ 3506 if (!strncmp(func->name, "__cfi_", 6) || 3507 !strncmp(func->name, "__pfx_", 6)) 3508 return 0; 3509 3510 WARN("%s() falls through to next function %s()", 3511 func->name, insn_func(insn)->name); 3512 return 1; 3513 } 3514 3515 if (func && insn->ignore) { 3516 WARN_FUNC("BUG: why am I validating an ignored function?", 3517 sec, insn->offset); 3518 return 1; 3519 } 3520 3521 visited = VISITED_BRANCH << state.uaccess; 3522 if (insn->visited & VISITED_BRANCH_MASK) { 3523 if (!insn->hint && !insn_cfi_match(insn, &state.cfi)) 3524 return 1; 3525 3526 if (insn->visited & visited) 3527 return 0; 3528 } else { 3529 nr_insns_visited++; 3530 } 3531 3532 if (state.noinstr) 3533 state.instr += insn->instr; 3534 3535 if (insn->hint) { 3536 if (insn->restore) { 3537 struct instruction *save_insn, *i; 3538 3539 i = insn; 3540 save_insn = NULL; 3541 3542 sym_for_each_insn_continue_reverse(file, func, i) { 3543 if (i->save) { 3544 save_insn = i; 3545 break; 3546 } 3547 } 3548 3549 if (!save_insn) { 3550 WARN_FUNC("no corresponding CFI save for CFI restore", 3551 sec, insn->offset); 3552 return 1; 3553 } 3554 3555 if (!save_insn->visited) { 3556 WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo", 3557 sec, insn->offset); 3558 return 1; 3559 } 3560 3561 insn->cfi = save_insn->cfi; 3562 nr_cfi_reused++; 3563 } 3564 3565 state.cfi = *insn->cfi; 3566 } else { 3567 /* XXX track if we actually changed state.cfi */ 3568 3569 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) { 3570 insn->cfi = prev_insn->cfi; 3571 nr_cfi_reused++; 3572 } else { 3573 insn->cfi = cfi_hash_find_or_add(&state.cfi); 3574 } 3575 } 3576 3577 insn->visited |= visited; 3578 3579 if (propagate_alt_cfi(file, insn)) 3580 return 1; 3581 3582 if (!insn->ignore_alts && !list_empty(&insn->alts)) { 3583 bool skip_orig = false; 3584 3585 list_for_each_entry(alt, &insn->alts, list) { 3586 if (alt->skip_orig) 3587 skip_orig = true; 3588 3589 ret = validate_branch(file, func, alt->insn, state); 3590 if (ret) { 3591 if (opts.backtrace) 3592 BT_FUNC("(alt)", insn); 3593 return ret; 3594 } 3595 } 3596 3597 if (skip_orig) 3598 return 0; 3599 } 3600 3601 if (handle_insn_ops(insn, next_insn, &state)) 3602 return 1; 3603 3604 switch (insn->type) { 3605 3606 case INSN_RETURN: 3607 return validate_return(func, insn, &state); 3608 3609 case INSN_CALL: 3610 case INSN_CALL_DYNAMIC: 3611 ret = validate_call(file, insn, &state); 3612 if (ret) 3613 return ret; 3614 3615 if (opts.stackval && func && !is_fentry_call(insn) && 3616 !has_valid_stack_frame(&state)) { 3617 WARN_FUNC("call without frame pointer save/setup", 3618 sec, insn->offset); 3619 return 1; 3620 } 3621 3622 if (insn->dead_end) 3623 return 0; 3624 3625 break; 3626 3627 case INSN_JUMP_CONDITIONAL: 3628 case INSN_JUMP_UNCONDITIONAL: 3629 if (is_sibling_call(insn)) { 3630 ret = validate_sibling_call(file, insn, &state); 3631 if (ret) 3632 return ret; 3633 3634 } else if (insn->jump_dest) { 3635 ret = validate_branch(file, func, 3636 insn->jump_dest, state); 3637 if (ret) { 3638 if (opts.backtrace) 3639 BT_FUNC("(branch)", insn); 3640 return ret; 3641 } 3642 } 3643 3644 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3645 return 0; 3646 3647 break; 3648 3649 case INSN_JUMP_DYNAMIC: 3650 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3651 if (is_sibling_call(insn)) { 3652 ret = validate_sibling_call(file, insn, &state); 3653 if (ret) 3654 return ret; 3655 } 3656 3657 if (insn->type == INSN_JUMP_DYNAMIC) 3658 return 0; 3659 3660 break; 3661 3662 case INSN_CONTEXT_SWITCH: 3663 if (func && (!next_insn || !next_insn->hint)) { 3664 WARN_FUNC("unsupported instruction in callable function", 3665 sec, insn->offset); 3666 return 1; 3667 } 3668 return 0; 3669 3670 case INSN_STAC: 3671 if (state.uaccess) { 3672 WARN_FUNC("recursive UACCESS enable", sec, insn->offset); 3673 return 1; 3674 } 3675 3676 state.uaccess = true; 3677 break; 3678 3679 case INSN_CLAC: 3680 if (!state.uaccess && func) { 3681 WARN_FUNC("redundant UACCESS disable", sec, insn->offset); 3682 return 1; 3683 } 3684 3685 if (func_uaccess_safe(func) && !state.uaccess_stack) { 3686 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset); 3687 return 1; 3688 } 3689 3690 state.uaccess = false; 3691 break; 3692 3693 case INSN_STD: 3694 if (state.df) { 3695 WARN_FUNC("recursive STD", sec, insn->offset); 3696 return 1; 3697 } 3698 3699 state.df = true; 3700 break; 3701 3702 case INSN_CLD: 3703 if (!state.df && func) { 3704 WARN_FUNC("redundant CLD", sec, insn->offset); 3705 return 1; 3706 } 3707 3708 state.df = false; 3709 break; 3710 3711 default: 3712 break; 3713 } 3714 3715 if (insn->dead_end) 3716 return 0; 3717 3718 if (!next_insn) { 3719 if (state.cfi.cfa.base == CFI_UNDEFINED) 3720 return 0; 3721 WARN("%s: unexpected end of section", sec->name); 3722 return 1; 3723 } 3724 3725 prev_insn = insn; 3726 insn = next_insn; 3727 } 3728 3729 return 0; 3730 } 3731 3732 static int validate_unwind_hints(struct objtool_file *file, struct section *sec) 3733 { 3734 struct instruction *insn; 3735 struct insn_state state; 3736 int ret, warnings = 0; 3737 3738 if (!file->hints) 3739 return 0; 3740 3741 init_insn_state(file, &state, sec); 3742 3743 if (sec) { 3744 insn = find_insn(file, sec, 0); 3745 if (!insn) 3746 return 0; 3747 } else { 3748 insn = list_first_entry(&file->insn_list, typeof(*insn), list); 3749 } 3750 3751 while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) { 3752 if (insn->hint && !insn->visited && !insn->ignore) { 3753 ret = validate_branch(file, insn_func(insn), insn, state); 3754 if (ret && opts.backtrace) 3755 BT_FUNC("<=== (hint)", insn); 3756 warnings += ret; 3757 } 3758 3759 insn = list_next_entry(insn, list); 3760 } 3761 3762 return warnings; 3763 } 3764 3765 /* 3766 * Validate rethunk entry constraint: must untrain RET before the first RET. 3767 * 3768 * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes 3769 * before an actual RET instruction. 3770 */ 3771 static int validate_entry(struct objtool_file *file, struct instruction *insn) 3772 { 3773 struct instruction *next, *dest; 3774 int ret, warnings = 0; 3775 3776 for (;;) { 3777 next = next_insn_to_validate(file, insn); 3778 3779 if (insn->visited & VISITED_ENTRY) 3780 return 0; 3781 3782 insn->visited |= VISITED_ENTRY; 3783 3784 if (!insn->ignore_alts && !list_empty(&insn->alts)) { 3785 struct alternative *alt; 3786 bool skip_orig = false; 3787 3788 list_for_each_entry(alt, &insn->alts, list) { 3789 if (alt->skip_orig) 3790 skip_orig = true; 3791 3792 ret = validate_entry(file, alt->insn); 3793 if (ret) { 3794 if (opts.backtrace) 3795 BT_FUNC("(alt)", insn); 3796 return ret; 3797 } 3798 } 3799 3800 if (skip_orig) 3801 return 0; 3802 } 3803 3804 switch (insn->type) { 3805 3806 case INSN_CALL_DYNAMIC: 3807 case INSN_JUMP_DYNAMIC: 3808 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3809 WARN_FUNC("early indirect call", insn->sec, insn->offset); 3810 return 1; 3811 3812 case INSN_JUMP_UNCONDITIONAL: 3813 case INSN_JUMP_CONDITIONAL: 3814 if (!is_sibling_call(insn)) { 3815 if (!insn->jump_dest) { 3816 WARN_FUNC("unresolved jump target after linking?!?", 3817 insn->sec, insn->offset); 3818 return -1; 3819 } 3820 ret = validate_entry(file, insn->jump_dest); 3821 if (ret) { 3822 if (opts.backtrace) { 3823 BT_FUNC("(branch%s)", insn, 3824 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : ""); 3825 } 3826 return ret; 3827 } 3828 3829 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3830 return 0; 3831 3832 break; 3833 } 3834 3835 /* fallthrough */ 3836 case INSN_CALL: 3837 dest = find_insn(file, insn->call_dest->sec, 3838 insn->call_dest->offset); 3839 if (!dest) { 3840 WARN("Unresolved function after linking!?: %s", 3841 insn->call_dest->name); 3842 return -1; 3843 } 3844 3845 ret = validate_entry(file, dest); 3846 if (ret) { 3847 if (opts.backtrace) 3848 BT_FUNC("(call)", insn); 3849 return ret; 3850 } 3851 /* 3852 * If a call returns without error, it must have seen UNTRAIN_RET. 3853 * Therefore any non-error return is a success. 3854 */ 3855 return 0; 3856 3857 case INSN_RETURN: 3858 WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset); 3859 return 1; 3860 3861 case INSN_NOP: 3862 if (insn->retpoline_safe) 3863 return 0; 3864 break; 3865 3866 default: 3867 break; 3868 } 3869 3870 if (!next) { 3871 WARN_FUNC("teh end!", insn->sec, insn->offset); 3872 return -1; 3873 } 3874 insn = next; 3875 } 3876 3877 return warnings; 3878 } 3879 3880 /* 3881 * Validate that all branches starting at 'insn->entry' encounter UNRET_END 3882 * before RET. 3883 */ 3884 static int validate_unret(struct objtool_file *file) 3885 { 3886 struct instruction *insn; 3887 int ret, warnings = 0; 3888 3889 for_each_insn(file, insn) { 3890 if (!insn->entry) 3891 continue; 3892 3893 ret = validate_entry(file, insn); 3894 if (ret < 0) { 3895 WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset); 3896 return ret; 3897 } 3898 warnings += ret; 3899 } 3900 3901 return warnings; 3902 } 3903 3904 static int validate_retpoline(struct objtool_file *file) 3905 { 3906 struct instruction *insn; 3907 int warnings = 0; 3908 3909 for_each_insn(file, insn) { 3910 if (insn->type != INSN_JUMP_DYNAMIC && 3911 insn->type != INSN_CALL_DYNAMIC && 3912 insn->type != INSN_RETURN) 3913 continue; 3914 3915 if (insn->retpoline_safe) 3916 continue; 3917 3918 if (insn->sec->init) 3919 continue; 3920 3921 if (insn->type == INSN_RETURN) { 3922 if (opts.rethunk) { 3923 WARN_FUNC("'naked' return found in RETHUNK build", 3924 insn->sec, insn->offset); 3925 } else 3926 continue; 3927 } else { 3928 WARN_FUNC("indirect %s found in RETPOLINE build", 3929 insn->sec, insn->offset, 3930 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); 3931 } 3932 3933 warnings++; 3934 } 3935 3936 return warnings; 3937 } 3938 3939 static bool is_kasan_insn(struct instruction *insn) 3940 { 3941 return (insn->type == INSN_CALL && 3942 !strcmp(insn->call_dest->name, "__asan_handle_no_return")); 3943 } 3944 3945 static bool is_ubsan_insn(struct instruction *insn) 3946 { 3947 return (insn->type == INSN_CALL && 3948 !strcmp(insn->call_dest->name, 3949 "__ubsan_handle_builtin_unreachable")); 3950 } 3951 3952 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) 3953 { 3954 int i; 3955 struct instruction *prev_insn; 3956 3957 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP) 3958 return true; 3959 3960 /* 3961 * Ignore alternative replacement instructions. This can happen 3962 * when a whitelisted function uses one of the ALTERNATIVE macros. 3963 */ 3964 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 3965 !strcmp(insn->sec->name, ".altinstr_aux")) 3966 return true; 3967 3968 /* 3969 * Whole archive runs might encounter dead code from weak symbols. 3970 * This is where the linker will have dropped the weak symbol in 3971 * favour of a regular symbol, but leaves the code in place. 3972 * 3973 * In this case we'll find a piece of code (whole function) that is not 3974 * covered by a !section symbol. Ignore them. 3975 */ 3976 if (opts.link && !insn_func(insn)) { 3977 int size = find_symbol_hole_containing(insn->sec, insn->offset); 3978 unsigned long end = insn->offset + size; 3979 3980 if (!size) /* not a hole */ 3981 return false; 3982 3983 if (size < 0) /* hole until the end */ 3984 return true; 3985 3986 sec_for_each_insn_continue(file, insn) { 3987 /* 3988 * If we reach a visited instruction at or before the 3989 * end of the hole, ignore the unreachable. 3990 */ 3991 if (insn->visited) 3992 return true; 3993 3994 if (insn->offset >= end) 3995 break; 3996 3997 /* 3998 * If this hole jumps to a .cold function, mark it ignore too. 3999 */ 4000 if (insn->jump_dest && insn_func(insn->jump_dest) && 4001 strstr(insn_func(insn->jump_dest)->name, ".cold")) { 4002 struct instruction *dest = insn->jump_dest; 4003 func_for_each_insn(file, insn_func(dest), dest) 4004 dest->ignore = true; 4005 } 4006 } 4007 4008 return false; 4009 } 4010 4011 if (!insn_func(insn)) 4012 return false; 4013 4014 if (insn_func(insn)->static_call_tramp) 4015 return true; 4016 4017 /* 4018 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees 4019 * __builtin_unreachable(). The BUG() macro has an unreachable() after 4020 * the UD2, which causes GCC's undefined trap logic to emit another UD2 4021 * (or occasionally a JMP to UD2). 4022 * 4023 * It may also insert a UD2 after calling a __noreturn function. 4024 */ 4025 prev_insn = list_prev_entry(insn, list); 4026 if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) && 4027 (insn->type == INSN_BUG || 4028 (insn->type == INSN_JUMP_UNCONDITIONAL && 4029 insn->jump_dest && insn->jump_dest->type == INSN_BUG))) 4030 return true; 4031 4032 /* 4033 * Check if this (or a subsequent) instruction is related to 4034 * CONFIG_UBSAN or CONFIG_KASAN. 4035 * 4036 * End the search at 5 instructions to avoid going into the weeds. 4037 */ 4038 for (i = 0; i < 5; i++) { 4039 4040 if (is_kasan_insn(insn) || is_ubsan_insn(insn)) 4041 return true; 4042 4043 if (insn->type == INSN_JUMP_UNCONDITIONAL) { 4044 if (insn->jump_dest && 4045 insn_func(insn->jump_dest) == insn_func(insn)) { 4046 insn = insn->jump_dest; 4047 continue; 4048 } 4049 4050 break; 4051 } 4052 4053 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len) 4054 break; 4055 4056 insn = list_next_entry(insn, list); 4057 } 4058 4059 return false; 4060 } 4061 4062 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func, 4063 struct instruction *insn) 4064 { 4065 if (!opts.prefix) 4066 return 0; 4067 4068 for (;;) { 4069 struct instruction *prev = list_prev_entry(insn, list); 4070 u64 offset; 4071 4072 if (&prev->list == &file->insn_list) 4073 break; 4074 4075 if (prev->type != INSN_NOP) 4076 break; 4077 4078 offset = func->offset - prev->offset; 4079 if (offset >= opts.prefix) { 4080 if (offset == opts.prefix) { 4081 /* 4082 * Since the sec->symbol_list is ordered by 4083 * offset (see elf_add_symbol()) the added 4084 * symbol will not be seen by the iteration in 4085 * validate_section(). 4086 * 4087 * Hence the lack of list_for_each_entry_safe() 4088 * there. 4089 * 4090 * The direct concequence is that prefix symbols 4091 * don't get visited (because pointless), except 4092 * for the logic in ignore_unreachable_insn() 4093 * that needs the terminating insn to be visited 4094 * otherwise it will report the hole. 4095 * 4096 * Hence mark the first instruction of the 4097 * prefix symbol as visisted. 4098 */ 4099 prev->visited |= VISITED_BRANCH; 4100 elf_create_prefix_symbol(file->elf, func, opts.prefix); 4101 } 4102 break; 4103 } 4104 insn = prev; 4105 } 4106 4107 return 0; 4108 } 4109 4110 static int validate_symbol(struct objtool_file *file, struct section *sec, 4111 struct symbol *sym, struct insn_state *state) 4112 { 4113 struct instruction *insn; 4114 int ret; 4115 4116 if (!sym->len) { 4117 WARN("%s() is missing an ELF size annotation", sym->name); 4118 return 1; 4119 } 4120 4121 if (sym->pfunc != sym || sym->alias != sym) 4122 return 0; 4123 4124 insn = find_insn(file, sec, sym->offset); 4125 if (!insn || insn->ignore || insn->visited) 4126 return 0; 4127 4128 add_prefix_symbol(file, sym, insn); 4129 4130 state->uaccess = sym->uaccess_safe; 4131 4132 ret = validate_branch(file, insn_func(insn), insn, *state); 4133 if (ret && opts.backtrace) 4134 BT_FUNC("<=== (sym)", insn); 4135 return ret; 4136 } 4137 4138 static int validate_section(struct objtool_file *file, struct section *sec) 4139 { 4140 struct insn_state state; 4141 struct symbol *func; 4142 int warnings = 0; 4143 4144 list_for_each_entry(func, &sec->symbol_list, list) { 4145 if (func->type != STT_FUNC) 4146 continue; 4147 4148 init_insn_state(file, &state, sec); 4149 set_func_state(&state.cfi); 4150 4151 warnings += validate_symbol(file, sec, func, &state); 4152 } 4153 4154 return warnings; 4155 } 4156 4157 static int validate_noinstr_sections(struct objtool_file *file) 4158 { 4159 struct section *sec; 4160 int warnings = 0; 4161 4162 sec = find_section_by_name(file->elf, ".noinstr.text"); 4163 if (sec) { 4164 warnings += validate_section(file, sec); 4165 warnings += validate_unwind_hints(file, sec); 4166 } 4167 4168 sec = find_section_by_name(file->elf, ".entry.text"); 4169 if (sec) { 4170 warnings += validate_section(file, sec); 4171 warnings += validate_unwind_hints(file, sec); 4172 } 4173 4174 return warnings; 4175 } 4176 4177 static int validate_functions(struct objtool_file *file) 4178 { 4179 struct section *sec; 4180 int warnings = 0; 4181 4182 for_each_sec(file, sec) { 4183 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 4184 continue; 4185 4186 warnings += validate_section(file, sec); 4187 } 4188 4189 return warnings; 4190 } 4191 4192 static void mark_endbr_used(struct instruction *insn) 4193 { 4194 if (!list_empty(&insn->call_node)) 4195 list_del_init(&insn->call_node); 4196 } 4197 4198 static bool noendbr_range(struct objtool_file *file, struct instruction *insn) 4199 { 4200 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1); 4201 struct instruction *first; 4202 4203 if (!sym) 4204 return false; 4205 4206 first = find_insn(file, sym->sec, sym->offset); 4207 if (!first) 4208 return false; 4209 4210 if (first->type != INSN_ENDBR && !first->noendbr) 4211 return false; 4212 4213 return insn->offset == sym->offset + sym->len; 4214 } 4215 4216 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn) 4217 { 4218 struct instruction *dest; 4219 struct reloc *reloc; 4220 unsigned long off; 4221 int warnings = 0; 4222 4223 /* 4224 * Looking for function pointer load relocations. Ignore 4225 * direct/indirect branches: 4226 */ 4227 switch (insn->type) { 4228 case INSN_CALL: 4229 case INSN_CALL_DYNAMIC: 4230 case INSN_JUMP_CONDITIONAL: 4231 case INSN_JUMP_UNCONDITIONAL: 4232 case INSN_JUMP_DYNAMIC: 4233 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4234 case INSN_RETURN: 4235 case INSN_NOP: 4236 return 0; 4237 default: 4238 break; 4239 } 4240 4241 for (reloc = insn_reloc(file, insn); 4242 reloc; 4243 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 4244 reloc->offset + 1, 4245 (insn->offset + insn->len) - (reloc->offset + 1))) { 4246 4247 /* 4248 * static_call_update() references the trampoline, which 4249 * doesn't have (or need) ENDBR. Skip warning in that case. 4250 */ 4251 if (reloc->sym->static_call_tramp) 4252 continue; 4253 4254 off = reloc->sym->offset; 4255 if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32) 4256 off += arch_dest_reloc_offset(reloc->addend); 4257 else 4258 off += reloc->addend; 4259 4260 dest = find_insn(file, reloc->sym->sec, off); 4261 if (!dest) 4262 continue; 4263 4264 if (dest->type == INSN_ENDBR) { 4265 mark_endbr_used(dest); 4266 continue; 4267 } 4268 4269 if (insn_func(dest) && insn_func(dest) == insn_func(insn)) { 4270 /* 4271 * Anything from->to self is either _THIS_IP_ or 4272 * IRET-to-self. 4273 * 4274 * There is no sane way to annotate _THIS_IP_ since the 4275 * compiler treats the relocation as a constant and is 4276 * happy to fold in offsets, skewing any annotation we 4277 * do, leading to vast amounts of false-positives. 4278 * 4279 * There's also compiler generated _THIS_IP_ through 4280 * KCOV and such which we have no hope of annotating. 4281 * 4282 * As such, blanket accept self-references without 4283 * issue. 4284 */ 4285 continue; 4286 } 4287 4288 /* 4289 * Accept anything ANNOTATE_NOENDBR. 4290 */ 4291 if (dest->noendbr) 4292 continue; 4293 4294 /* 4295 * Accept if this is the instruction after a symbol 4296 * that is (no)endbr -- typical code-range usage. 4297 */ 4298 if (noendbr_range(file, dest)) 4299 continue; 4300 4301 WARN_FUNC("relocation to !ENDBR: %s", 4302 insn->sec, insn->offset, 4303 offstr(dest->sec, dest->offset)); 4304 4305 warnings++; 4306 } 4307 4308 return warnings; 4309 } 4310 4311 static int validate_ibt_data_reloc(struct objtool_file *file, 4312 struct reloc *reloc) 4313 { 4314 struct instruction *dest; 4315 4316 dest = find_insn(file, reloc->sym->sec, 4317 reloc->sym->offset + reloc->addend); 4318 if (!dest) 4319 return 0; 4320 4321 if (dest->type == INSN_ENDBR) { 4322 mark_endbr_used(dest); 4323 return 0; 4324 } 4325 4326 if (dest->noendbr) 4327 return 0; 4328 4329 WARN_FUNC("data relocation to !ENDBR: %s", 4330 reloc->sec->base, reloc->offset, 4331 offstr(dest->sec, dest->offset)); 4332 4333 return 1; 4334 } 4335 4336 /* 4337 * Validate IBT rules and remove used ENDBR instructions from the seal list. 4338 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with 4339 * NOPs) later, in create_ibt_endbr_seal_sections(). 4340 */ 4341 static int validate_ibt(struct objtool_file *file) 4342 { 4343 struct section *sec; 4344 struct reloc *reloc; 4345 struct instruction *insn; 4346 int warnings = 0; 4347 4348 for_each_insn(file, insn) 4349 warnings += validate_ibt_insn(file, insn); 4350 4351 for_each_sec(file, sec) { 4352 4353 /* Already done by validate_ibt_insn() */ 4354 if (sec->sh.sh_flags & SHF_EXECINSTR) 4355 continue; 4356 4357 if (!sec->reloc) 4358 continue; 4359 4360 /* 4361 * These sections can reference text addresses, but not with 4362 * the intent to indirect branch to them. 4363 */ 4364 if ((!strncmp(sec->name, ".discard", 8) && 4365 strcmp(sec->name, ".discard.ibt_endbr_noseal")) || 4366 !strncmp(sec->name, ".debug", 6) || 4367 !strcmp(sec->name, ".altinstructions") || 4368 !strcmp(sec->name, ".ibt_endbr_seal") || 4369 !strcmp(sec->name, ".orc_unwind_ip") || 4370 !strcmp(sec->name, ".parainstructions") || 4371 !strcmp(sec->name, ".retpoline_sites") || 4372 !strcmp(sec->name, ".smp_locks") || 4373 !strcmp(sec->name, ".static_call_sites") || 4374 !strcmp(sec->name, "_error_injection_whitelist") || 4375 !strcmp(sec->name, "_kprobe_blacklist") || 4376 !strcmp(sec->name, "__bug_table") || 4377 !strcmp(sec->name, "__ex_table") || 4378 !strcmp(sec->name, "__jump_table") || 4379 !strcmp(sec->name, "__mcount_loc") || 4380 !strcmp(sec->name, ".kcfi_traps") || 4381 strstr(sec->name, "__patchable_function_entries")) 4382 continue; 4383 4384 list_for_each_entry(reloc, &sec->reloc->reloc_list, list) 4385 warnings += validate_ibt_data_reloc(file, reloc); 4386 } 4387 4388 return warnings; 4389 } 4390 4391 static int validate_sls(struct objtool_file *file) 4392 { 4393 struct instruction *insn, *next_insn; 4394 int warnings = 0; 4395 4396 for_each_insn(file, insn) { 4397 next_insn = next_insn_same_sec(file, insn); 4398 4399 if (insn->retpoline_safe) 4400 continue; 4401 4402 switch (insn->type) { 4403 case INSN_RETURN: 4404 if (!next_insn || next_insn->type != INSN_TRAP) { 4405 WARN_FUNC("missing int3 after ret", 4406 insn->sec, insn->offset); 4407 warnings++; 4408 } 4409 4410 break; 4411 case INSN_JUMP_DYNAMIC: 4412 if (!next_insn || next_insn->type != INSN_TRAP) { 4413 WARN_FUNC("missing int3 after indirect jump", 4414 insn->sec, insn->offset); 4415 warnings++; 4416 } 4417 break; 4418 default: 4419 break; 4420 } 4421 } 4422 4423 return warnings; 4424 } 4425 4426 static int validate_reachable_instructions(struct objtool_file *file) 4427 { 4428 struct instruction *insn; 4429 4430 if (file->ignore_unreachables) 4431 return 0; 4432 4433 for_each_insn(file, insn) { 4434 if (insn->visited || ignore_unreachable_insn(file, insn)) 4435 continue; 4436 4437 WARN_FUNC("unreachable instruction", insn->sec, insn->offset); 4438 return 1; 4439 } 4440 4441 return 0; 4442 } 4443 4444 int check(struct objtool_file *file) 4445 { 4446 int ret, warnings = 0; 4447 4448 arch_initial_func_cfi_state(&initial_func_cfi); 4449 init_cfi_state(&init_cfi); 4450 init_cfi_state(&func_cfi); 4451 set_func_state(&func_cfi); 4452 4453 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) 4454 goto out; 4455 4456 cfi_hash_add(&init_cfi); 4457 cfi_hash_add(&func_cfi); 4458 4459 ret = decode_sections(file); 4460 if (ret < 0) 4461 goto out; 4462 4463 warnings += ret; 4464 4465 if (list_empty(&file->insn_list)) 4466 goto out; 4467 4468 if (opts.retpoline) { 4469 ret = validate_retpoline(file); 4470 if (ret < 0) 4471 return ret; 4472 warnings += ret; 4473 } 4474 4475 if (opts.stackval || opts.orc || opts.uaccess) { 4476 ret = validate_functions(file); 4477 if (ret < 0) 4478 goto out; 4479 warnings += ret; 4480 4481 ret = validate_unwind_hints(file, NULL); 4482 if (ret < 0) 4483 goto out; 4484 warnings += ret; 4485 4486 if (!warnings) { 4487 ret = validate_reachable_instructions(file); 4488 if (ret < 0) 4489 goto out; 4490 warnings += ret; 4491 } 4492 4493 } else if (opts.noinstr) { 4494 ret = validate_noinstr_sections(file); 4495 if (ret < 0) 4496 goto out; 4497 warnings += ret; 4498 } 4499 4500 if (opts.unret) { 4501 /* 4502 * Must be after validate_branch() and friends, it plays 4503 * further games with insn->visited. 4504 */ 4505 ret = validate_unret(file); 4506 if (ret < 0) 4507 return ret; 4508 warnings += ret; 4509 } 4510 4511 if (opts.ibt) { 4512 ret = validate_ibt(file); 4513 if (ret < 0) 4514 goto out; 4515 warnings += ret; 4516 } 4517 4518 if (opts.sls) { 4519 ret = validate_sls(file); 4520 if (ret < 0) 4521 goto out; 4522 warnings += ret; 4523 } 4524 4525 if (opts.static_call) { 4526 ret = create_static_call_sections(file); 4527 if (ret < 0) 4528 goto out; 4529 warnings += ret; 4530 } 4531 4532 if (opts.retpoline) { 4533 ret = create_retpoline_sites_sections(file); 4534 if (ret < 0) 4535 goto out; 4536 warnings += ret; 4537 } 4538 4539 if (opts.cfi) { 4540 ret = create_cfi_sections(file); 4541 if (ret < 0) 4542 goto out; 4543 warnings += ret; 4544 } 4545 4546 if (opts.rethunk) { 4547 ret = create_return_sites_sections(file); 4548 if (ret < 0) 4549 goto out; 4550 warnings += ret; 4551 4552 if (opts.hack_skylake) { 4553 ret = create_direct_call_sections(file); 4554 if (ret < 0) 4555 goto out; 4556 warnings += ret; 4557 } 4558 } 4559 4560 if (opts.mcount) { 4561 ret = create_mcount_loc_sections(file); 4562 if (ret < 0) 4563 goto out; 4564 warnings += ret; 4565 } 4566 4567 if (opts.ibt) { 4568 ret = create_ibt_endbr_seal_sections(file); 4569 if (ret < 0) 4570 goto out; 4571 warnings += ret; 4572 } 4573 4574 if (opts.orc && !list_empty(&file->insn_list)) { 4575 ret = orc_create(file); 4576 if (ret < 0) 4577 goto out; 4578 warnings += ret; 4579 } 4580 4581 4582 if (opts.stats) { 4583 printf("nr_insns_visited: %ld\n", nr_insns_visited); 4584 printf("nr_cfi: %ld\n", nr_cfi); 4585 printf("nr_cfi_reused: %ld\n", nr_cfi_reused); 4586 printf("nr_cfi_cache: %ld\n", nr_cfi_cache); 4587 } 4588 4589 out: 4590 /* 4591 * For now, don't fail the kernel build on fatal warnings. These 4592 * errors are still fairly common due to the growing matrix of 4593 * supported toolchains and their recent pace of change. 4594 */ 4595 return 0; 4596 } 4597