1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6 #include <string.h> 7 #include <stdlib.h> 8 #include <inttypes.h> 9 #include <sys/mman.h> 10 11 #include <arch/elf.h> 12 #include <objtool/builtin.h> 13 #include <objtool/cfi.h> 14 #include <objtool/arch.h> 15 #include <objtool/check.h> 16 #include <objtool/special.h> 17 #include <objtool/warn.h> 18 #include <objtool/endianness.h> 19 20 #include <linux/objtool.h> 21 #include <linux/hashtable.h> 22 #include <linux/kernel.h> 23 #include <linux/static_call_types.h> 24 25 struct alternative { 26 struct list_head list; 27 struct instruction *insn; 28 bool skip_orig; 29 }; 30 31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 32 33 static struct cfi_init_state initial_func_cfi; 34 static struct cfi_state init_cfi; 35 static struct cfi_state func_cfi; 36 37 struct instruction *find_insn(struct objtool_file *file, 38 struct section *sec, unsigned long offset) 39 { 40 struct instruction *insn; 41 42 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) { 43 if (insn->sec == sec && insn->offset == offset) 44 return insn; 45 } 46 47 return NULL; 48 } 49 50 static struct instruction *next_insn_same_sec(struct objtool_file *file, 51 struct instruction *insn) 52 { 53 struct instruction *next = list_next_entry(insn, list); 54 55 if (!next || &next->list == &file->insn_list || next->sec != insn->sec) 56 return NULL; 57 58 return next; 59 } 60 61 static struct instruction *next_insn_same_func(struct objtool_file *file, 62 struct instruction *insn) 63 { 64 struct instruction *next = list_next_entry(insn, list); 65 struct symbol *func = insn->func; 66 67 if (!func) 68 return NULL; 69 70 if (&next->list != &file->insn_list && next->func == func) 71 return next; 72 73 /* Check if we're already in the subfunction: */ 74 if (func == func->cfunc) 75 return NULL; 76 77 /* Move to the subfunction: */ 78 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 79 } 80 81 static struct instruction *prev_insn_same_sym(struct objtool_file *file, 82 struct instruction *insn) 83 { 84 struct instruction *prev = list_prev_entry(insn, list); 85 86 if (&prev->list != &file->insn_list && prev->func == insn->func) 87 return prev; 88 89 return NULL; 90 } 91 92 #define func_for_each_insn(file, func, insn) \ 93 for (insn = find_insn(file, func->sec, func->offset); \ 94 insn; \ 95 insn = next_insn_same_func(file, insn)) 96 97 #define sym_for_each_insn(file, sym, insn) \ 98 for (insn = find_insn(file, sym->sec, sym->offset); \ 99 insn && &insn->list != &file->insn_list && \ 100 insn->sec == sym->sec && \ 101 insn->offset < sym->offset + sym->len; \ 102 insn = list_next_entry(insn, list)) 103 104 #define sym_for_each_insn_continue_reverse(file, sym, insn) \ 105 for (insn = list_prev_entry(insn, list); \ 106 &insn->list != &file->insn_list && \ 107 insn->sec == sym->sec && insn->offset >= sym->offset; \ 108 insn = list_prev_entry(insn, list)) 109 110 #define sec_for_each_insn_from(file, insn) \ 111 for (; insn; insn = next_insn_same_sec(file, insn)) 112 113 #define sec_for_each_insn_continue(file, insn) \ 114 for (insn = next_insn_same_sec(file, insn); insn; \ 115 insn = next_insn_same_sec(file, insn)) 116 117 static bool is_jump_table_jump(struct instruction *insn) 118 { 119 struct alt_group *alt_group = insn->alt_group; 120 121 if (insn->jump_table) 122 return true; 123 124 /* Retpoline alternative for a jump table? */ 125 return alt_group && alt_group->orig_group && 126 alt_group->orig_group->first_insn->jump_table; 127 } 128 129 static bool is_sibling_call(struct instruction *insn) 130 { 131 /* 132 * Assume only ELF functions can make sibling calls. This ensures 133 * sibling call detection consistency between vmlinux.o and individual 134 * objects. 135 */ 136 if (!insn->func) 137 return false; 138 139 /* An indirect jump is either a sibling call or a jump to a table. */ 140 if (insn->type == INSN_JUMP_DYNAMIC) 141 return !is_jump_table_jump(insn); 142 143 /* add_jump_destinations() sets insn->call_dest for sibling calls. */ 144 return (is_static_jump(insn) && insn->call_dest); 145 } 146 147 /* 148 * This checks to see if the given function is a "noreturn" function. 149 * 150 * For global functions which are outside the scope of this object file, we 151 * have to keep a manual list of them. 152 * 153 * For local functions, we have to detect them manually by simply looking for 154 * the lack of a return instruction. 155 */ 156 static bool __dead_end_function(struct objtool_file *file, struct symbol *func, 157 int recursion) 158 { 159 int i; 160 struct instruction *insn; 161 bool empty = true; 162 163 /* 164 * Unfortunately these have to be hard coded because the noreturn 165 * attribute isn't provided in ELF data. Keep 'em sorted. 166 */ 167 static const char * const global_noreturns[] = { 168 "__invalid_creds", 169 "__module_put_and_kthread_exit", 170 "__reiserfs_panic", 171 "__stack_chk_fail", 172 "__ubsan_handle_builtin_unreachable", 173 "cpu_bringup_and_idle", 174 "cpu_startup_entry", 175 "do_exit", 176 "do_group_exit", 177 "do_task_dead", 178 "ex_handler_msr_mce", 179 "fortify_panic", 180 "kthread_complete_and_exit", 181 "kthread_exit", 182 "kunit_try_catch_throw", 183 "lbug_with_loc", 184 "machine_real_restart", 185 "make_task_dead", 186 "panic", 187 "rewind_stack_and_make_dead", 188 "sev_es_terminate", 189 "snp_abort", 190 "stop_this_cpu", 191 "usercopy_abort", 192 "xen_start_kernel", 193 }; 194 195 if (!func) 196 return false; 197 198 if (func->bind == STB_WEAK) 199 return false; 200 201 if (func->bind == STB_GLOBAL) 202 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) 203 if (!strcmp(func->name, global_noreturns[i])) 204 return true; 205 206 if (!func->len) 207 return false; 208 209 insn = find_insn(file, func->sec, func->offset); 210 if (!insn->func) 211 return false; 212 213 func_for_each_insn(file, func, insn) { 214 empty = false; 215 216 if (insn->type == INSN_RETURN) 217 return false; 218 } 219 220 if (empty) 221 return false; 222 223 /* 224 * A function can have a sibling call instead of a return. In that 225 * case, the function's dead-end status depends on whether the target 226 * of the sibling call returns. 227 */ 228 func_for_each_insn(file, func, insn) { 229 if (is_sibling_call(insn)) { 230 struct instruction *dest = insn->jump_dest; 231 232 if (!dest) 233 /* sibling call to another file */ 234 return false; 235 236 /* local sibling call */ 237 if (recursion == 5) { 238 /* 239 * Infinite recursion: two functions have 240 * sibling calls to each other. This is a very 241 * rare case. It means they aren't dead ends. 242 */ 243 return false; 244 } 245 246 return __dead_end_function(file, dest->func, recursion+1); 247 } 248 } 249 250 return true; 251 } 252 253 static bool dead_end_function(struct objtool_file *file, struct symbol *func) 254 { 255 return __dead_end_function(file, func, 0); 256 } 257 258 static void init_cfi_state(struct cfi_state *cfi) 259 { 260 int i; 261 262 for (i = 0; i < CFI_NUM_REGS; i++) { 263 cfi->regs[i].base = CFI_UNDEFINED; 264 cfi->vals[i].base = CFI_UNDEFINED; 265 } 266 cfi->cfa.base = CFI_UNDEFINED; 267 cfi->drap_reg = CFI_UNDEFINED; 268 cfi->drap_offset = -1; 269 } 270 271 static void init_insn_state(struct objtool_file *file, struct insn_state *state, 272 struct section *sec) 273 { 274 memset(state, 0, sizeof(*state)); 275 init_cfi_state(&state->cfi); 276 277 /* 278 * We need the full vmlinux for noinstr validation, otherwise we can 279 * not correctly determine insn->call_dest->sec (external symbols do 280 * not have a section). 281 */ 282 if (opts.link && opts.noinstr && sec) 283 state->noinstr = sec->noinstr; 284 } 285 286 static struct cfi_state *cfi_alloc(void) 287 { 288 struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1); 289 if (!cfi) { 290 WARN("calloc failed"); 291 exit(1); 292 } 293 nr_cfi++; 294 return cfi; 295 } 296 297 static int cfi_bits; 298 static struct hlist_head *cfi_hash; 299 300 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2) 301 { 302 return memcmp((void *)cfi1 + sizeof(cfi1->hash), 303 (void *)cfi2 + sizeof(cfi2->hash), 304 sizeof(struct cfi_state) - sizeof(struct hlist_node)); 305 } 306 307 static inline u32 cfi_key(struct cfi_state *cfi) 308 { 309 return jhash((void *)cfi + sizeof(cfi->hash), 310 sizeof(*cfi) - sizeof(cfi->hash), 0); 311 } 312 313 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi) 314 { 315 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 316 struct cfi_state *obj; 317 318 hlist_for_each_entry(obj, head, hash) { 319 if (!cficmp(cfi, obj)) { 320 nr_cfi_cache++; 321 return obj; 322 } 323 } 324 325 obj = cfi_alloc(); 326 *obj = *cfi; 327 hlist_add_head(&obj->hash, head); 328 329 return obj; 330 } 331 332 static void cfi_hash_add(struct cfi_state *cfi) 333 { 334 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 335 336 hlist_add_head(&cfi->hash, head); 337 } 338 339 static void *cfi_hash_alloc(unsigned long size) 340 { 341 cfi_bits = max(10, ilog2(size)); 342 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits, 343 PROT_READ|PROT_WRITE, 344 MAP_PRIVATE|MAP_ANON, -1, 0); 345 if (cfi_hash == (void *)-1L) { 346 WARN("mmap fail cfi_hash"); 347 cfi_hash = NULL; 348 } else if (opts.stats) { 349 printf("cfi_bits: %d\n", cfi_bits); 350 } 351 352 return cfi_hash; 353 } 354 355 static unsigned long nr_insns; 356 static unsigned long nr_insns_visited; 357 358 /* 359 * Call the arch-specific instruction decoder for all the instructions and add 360 * them to the global instruction list. 361 */ 362 static int decode_instructions(struct objtool_file *file) 363 { 364 struct section *sec; 365 struct symbol *func; 366 unsigned long offset; 367 struct instruction *insn; 368 int ret; 369 370 for_each_sec(file, sec) { 371 372 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 373 continue; 374 375 if (strcmp(sec->name, ".altinstr_replacement") && 376 strcmp(sec->name, ".altinstr_aux") && 377 strncmp(sec->name, ".discard.", 9)) 378 sec->text = true; 379 380 if (!strcmp(sec->name, ".noinstr.text") || 381 !strcmp(sec->name, ".entry.text") || 382 !strncmp(sec->name, ".text.__x86.", 12)) 383 sec->noinstr = true; 384 385 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) { 386 insn = malloc(sizeof(*insn)); 387 if (!insn) { 388 WARN("malloc failed"); 389 return -1; 390 } 391 memset(insn, 0, sizeof(*insn)); 392 INIT_LIST_HEAD(&insn->alts); 393 INIT_LIST_HEAD(&insn->stack_ops); 394 INIT_LIST_HEAD(&insn->call_node); 395 396 insn->sec = sec; 397 insn->offset = offset; 398 399 ret = arch_decode_instruction(file, sec, offset, 400 sec->sh.sh_size - offset, 401 &insn->len, &insn->type, 402 &insn->immediate, 403 &insn->stack_ops); 404 if (ret) 405 goto err; 406 407 /* 408 * By default, "ud2" is a dead end unless otherwise 409 * annotated, because GCC 7 inserts it for certain 410 * divide-by-zero cases. 411 */ 412 if (insn->type == INSN_BUG) 413 insn->dead_end = true; 414 415 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset)); 416 list_add_tail(&insn->list, &file->insn_list); 417 nr_insns++; 418 } 419 420 list_for_each_entry(func, &sec->symbol_list, list) { 421 if (func->type != STT_FUNC || func->alias != func) 422 continue; 423 424 if (!find_insn(file, sec, func->offset)) { 425 WARN("%s(): can't find starting instruction", 426 func->name); 427 return -1; 428 } 429 430 sym_for_each_insn(file, func, insn) { 431 insn->func = func; 432 if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) { 433 if (insn->offset == insn->func->offset) { 434 list_add_tail(&insn->call_node, &file->endbr_list); 435 file->nr_endbr++; 436 } else { 437 file->nr_endbr_int++; 438 } 439 } 440 } 441 } 442 } 443 444 if (opts.stats) 445 printf("nr_insns: %lu\n", nr_insns); 446 447 return 0; 448 449 err: 450 free(insn); 451 return ret; 452 } 453 454 /* 455 * Read the pv_ops[] .data table to find the static initialized values. 456 */ 457 static int add_pv_ops(struct objtool_file *file, const char *symname) 458 { 459 struct symbol *sym, *func; 460 unsigned long off, end; 461 struct reloc *rel; 462 int idx; 463 464 sym = find_symbol_by_name(file->elf, symname); 465 if (!sym) 466 return 0; 467 468 off = sym->offset; 469 end = off + sym->len; 470 for (;;) { 471 rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 472 if (!rel) 473 break; 474 475 func = rel->sym; 476 if (func->type == STT_SECTION) 477 func = find_symbol_by_offset(rel->sym->sec, rel->addend); 478 479 idx = (rel->offset - sym->offset) / sizeof(unsigned long); 480 481 objtool_pv_add(file, idx, func); 482 483 off = rel->offset + 1; 484 if (off > end) 485 break; 486 } 487 488 return 0; 489 } 490 491 /* 492 * Allocate and initialize file->pv_ops[]. 493 */ 494 static int init_pv_ops(struct objtool_file *file) 495 { 496 static const char *pv_ops_tables[] = { 497 "pv_ops", 498 "xen_cpu_ops", 499 "xen_irq_ops", 500 "xen_mmu_ops", 501 NULL, 502 }; 503 const char *pv_ops; 504 struct symbol *sym; 505 int idx, nr; 506 507 if (!opts.noinstr) 508 return 0; 509 510 file->pv_ops = NULL; 511 512 sym = find_symbol_by_name(file->elf, "pv_ops"); 513 if (!sym) 514 return 0; 515 516 nr = sym->len / sizeof(unsigned long); 517 file->pv_ops = calloc(sizeof(struct pv_state), nr); 518 if (!file->pv_ops) 519 return -1; 520 521 for (idx = 0; idx < nr; idx++) 522 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 523 524 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) 525 add_pv_ops(file, pv_ops); 526 527 return 0; 528 } 529 530 static struct instruction *find_last_insn(struct objtool_file *file, 531 struct section *sec) 532 { 533 struct instruction *insn = NULL; 534 unsigned int offset; 535 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0; 536 537 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--) 538 insn = find_insn(file, sec, offset); 539 540 return insn; 541 } 542 543 /* 544 * Mark "ud2" instructions and manually annotated dead ends. 545 */ 546 static int add_dead_ends(struct objtool_file *file) 547 { 548 struct section *sec; 549 struct reloc *reloc; 550 struct instruction *insn; 551 552 /* 553 * Check for manually annotated dead ends. 554 */ 555 sec = find_section_by_name(file->elf, ".rela.discard.unreachable"); 556 if (!sec) 557 goto reachable; 558 559 list_for_each_entry(reloc, &sec->reloc_list, list) { 560 if (reloc->sym->type != STT_SECTION) { 561 WARN("unexpected relocation symbol type in %s", sec->name); 562 return -1; 563 } 564 insn = find_insn(file, reloc->sym->sec, reloc->addend); 565 if (insn) 566 insn = list_prev_entry(insn, list); 567 else if (reloc->addend == reloc->sym->sec->sh.sh_size) { 568 insn = find_last_insn(file, reloc->sym->sec); 569 if (!insn) { 570 WARN("can't find unreachable insn at %s+0x%" PRIx64, 571 reloc->sym->sec->name, reloc->addend); 572 return -1; 573 } 574 } else { 575 WARN("can't find unreachable insn at %s+0x%" PRIx64, 576 reloc->sym->sec->name, reloc->addend); 577 return -1; 578 } 579 580 insn->dead_end = true; 581 } 582 583 reachable: 584 /* 585 * These manually annotated reachable checks are needed for GCC 4.4, 586 * where the Linux unreachable() macro isn't supported. In that case 587 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's 588 * not a dead end. 589 */ 590 sec = find_section_by_name(file->elf, ".rela.discard.reachable"); 591 if (!sec) 592 return 0; 593 594 list_for_each_entry(reloc, &sec->reloc_list, list) { 595 if (reloc->sym->type != STT_SECTION) { 596 WARN("unexpected relocation symbol type in %s", sec->name); 597 return -1; 598 } 599 insn = find_insn(file, reloc->sym->sec, reloc->addend); 600 if (insn) 601 insn = list_prev_entry(insn, list); 602 else if (reloc->addend == reloc->sym->sec->sh.sh_size) { 603 insn = find_last_insn(file, reloc->sym->sec); 604 if (!insn) { 605 WARN("can't find reachable insn at %s+0x%" PRIx64, 606 reloc->sym->sec->name, reloc->addend); 607 return -1; 608 } 609 } else { 610 WARN("can't find reachable insn at %s+0x%" PRIx64, 611 reloc->sym->sec->name, reloc->addend); 612 return -1; 613 } 614 615 insn->dead_end = false; 616 } 617 618 return 0; 619 } 620 621 static int create_static_call_sections(struct objtool_file *file) 622 { 623 struct section *sec; 624 struct static_call_site *site; 625 struct instruction *insn; 626 struct symbol *key_sym; 627 char *key_name, *tmp; 628 int idx; 629 630 sec = find_section_by_name(file->elf, ".static_call_sites"); 631 if (sec) { 632 INIT_LIST_HEAD(&file->static_call_list); 633 WARN("file already has .static_call_sites section, skipping"); 634 return 0; 635 } 636 637 if (list_empty(&file->static_call_list)) 638 return 0; 639 640 idx = 0; 641 list_for_each_entry(insn, &file->static_call_list, call_node) 642 idx++; 643 644 sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE, 645 sizeof(struct static_call_site), idx); 646 if (!sec) 647 return -1; 648 649 idx = 0; 650 list_for_each_entry(insn, &file->static_call_list, call_node) { 651 652 site = (struct static_call_site *)sec->data->d_buf + idx; 653 memset(site, 0, sizeof(struct static_call_site)); 654 655 /* populate reloc for 'addr' */ 656 if (elf_add_reloc_to_insn(file->elf, sec, 657 idx * sizeof(struct static_call_site), 658 R_X86_64_PC32, 659 insn->sec, insn->offset)) 660 return -1; 661 662 /* find key symbol */ 663 key_name = strdup(insn->call_dest->name); 664 if (!key_name) { 665 perror("strdup"); 666 return -1; 667 } 668 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR, 669 STATIC_CALL_TRAMP_PREFIX_LEN)) { 670 WARN("static_call: trampoline name malformed: %s", key_name); 671 return -1; 672 } 673 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN; 674 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN); 675 676 key_sym = find_symbol_by_name(file->elf, tmp); 677 if (!key_sym) { 678 if (!opts.module) { 679 WARN("static_call: can't find static_call_key symbol: %s", tmp); 680 return -1; 681 } 682 683 /* 684 * For modules(), the key might not be exported, which 685 * means the module can make static calls but isn't 686 * allowed to change them. 687 * 688 * In that case we temporarily set the key to be the 689 * trampoline address. This is fixed up in 690 * static_call_add_module(). 691 */ 692 key_sym = insn->call_dest; 693 } 694 free(key_name); 695 696 /* populate reloc for 'key' */ 697 if (elf_add_reloc(file->elf, sec, 698 idx * sizeof(struct static_call_site) + 4, 699 R_X86_64_PC32, key_sym, 700 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL)) 701 return -1; 702 703 idx++; 704 } 705 706 return 0; 707 } 708 709 static int create_retpoline_sites_sections(struct objtool_file *file) 710 { 711 struct instruction *insn; 712 struct section *sec; 713 int idx; 714 715 sec = find_section_by_name(file->elf, ".retpoline_sites"); 716 if (sec) { 717 WARN("file already has .retpoline_sites, skipping"); 718 return 0; 719 } 720 721 idx = 0; 722 list_for_each_entry(insn, &file->retpoline_call_list, call_node) 723 idx++; 724 725 if (!idx) 726 return 0; 727 728 sec = elf_create_section(file->elf, ".retpoline_sites", 0, 729 sizeof(int), idx); 730 if (!sec) { 731 WARN("elf_create_section: .retpoline_sites"); 732 return -1; 733 } 734 735 idx = 0; 736 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 737 738 int *site = (int *)sec->data->d_buf + idx; 739 *site = 0; 740 741 if (elf_add_reloc_to_insn(file->elf, sec, 742 idx * sizeof(int), 743 R_X86_64_PC32, 744 insn->sec, insn->offset)) { 745 WARN("elf_add_reloc_to_insn: .retpoline_sites"); 746 return -1; 747 } 748 749 idx++; 750 } 751 752 return 0; 753 } 754 755 static int create_return_sites_sections(struct objtool_file *file) 756 { 757 struct instruction *insn; 758 struct section *sec; 759 int idx; 760 761 sec = find_section_by_name(file->elf, ".return_sites"); 762 if (sec) { 763 WARN("file already has .return_sites, skipping"); 764 return 0; 765 } 766 767 idx = 0; 768 list_for_each_entry(insn, &file->return_thunk_list, call_node) 769 idx++; 770 771 if (!idx) 772 return 0; 773 774 sec = elf_create_section(file->elf, ".return_sites", 0, 775 sizeof(int), idx); 776 if (!sec) { 777 WARN("elf_create_section: .return_sites"); 778 return -1; 779 } 780 781 idx = 0; 782 list_for_each_entry(insn, &file->return_thunk_list, call_node) { 783 784 int *site = (int *)sec->data->d_buf + idx; 785 *site = 0; 786 787 if (elf_add_reloc_to_insn(file->elf, sec, 788 idx * sizeof(int), 789 R_X86_64_PC32, 790 insn->sec, insn->offset)) { 791 WARN("elf_add_reloc_to_insn: .return_sites"); 792 return -1; 793 } 794 795 idx++; 796 } 797 798 return 0; 799 } 800 801 static int create_ibt_endbr_seal_sections(struct objtool_file *file) 802 { 803 struct instruction *insn; 804 struct section *sec; 805 int idx; 806 807 sec = find_section_by_name(file->elf, ".ibt_endbr_seal"); 808 if (sec) { 809 WARN("file already has .ibt_endbr_seal, skipping"); 810 return 0; 811 } 812 813 idx = 0; 814 list_for_each_entry(insn, &file->endbr_list, call_node) 815 idx++; 816 817 if (opts.stats) { 818 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); 819 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); 820 printf("ibt: superfluous ENDBR: %d\n", idx); 821 } 822 823 if (!idx) 824 return 0; 825 826 sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0, 827 sizeof(int), idx); 828 if (!sec) { 829 WARN("elf_create_section: .ibt_endbr_seal"); 830 return -1; 831 } 832 833 idx = 0; 834 list_for_each_entry(insn, &file->endbr_list, call_node) { 835 836 int *site = (int *)sec->data->d_buf + idx; 837 *site = 0; 838 839 if (elf_add_reloc_to_insn(file->elf, sec, 840 idx * sizeof(int), 841 R_X86_64_PC32, 842 insn->sec, insn->offset)) { 843 WARN("elf_add_reloc_to_insn: .ibt_endbr_seal"); 844 return -1; 845 } 846 847 idx++; 848 } 849 850 return 0; 851 } 852 853 static int create_mcount_loc_sections(struct objtool_file *file) 854 { 855 struct section *sec; 856 unsigned long *loc; 857 struct instruction *insn; 858 int idx; 859 860 sec = find_section_by_name(file->elf, "__mcount_loc"); 861 if (sec) { 862 INIT_LIST_HEAD(&file->mcount_loc_list); 863 WARN("file already has __mcount_loc section, skipping"); 864 return 0; 865 } 866 867 if (list_empty(&file->mcount_loc_list)) 868 return 0; 869 870 idx = 0; 871 list_for_each_entry(insn, &file->mcount_loc_list, call_node) 872 idx++; 873 874 sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx); 875 if (!sec) 876 return -1; 877 878 idx = 0; 879 list_for_each_entry(insn, &file->mcount_loc_list, call_node) { 880 881 loc = (unsigned long *)sec->data->d_buf + idx; 882 memset(loc, 0, sizeof(unsigned long)); 883 884 if (elf_add_reloc_to_insn(file->elf, sec, 885 idx * sizeof(unsigned long), 886 R_X86_64_64, 887 insn->sec, insn->offset)) 888 return -1; 889 890 idx++; 891 } 892 893 return 0; 894 } 895 896 /* 897 * Warnings shouldn't be reported for ignored functions. 898 */ 899 static void add_ignores(struct objtool_file *file) 900 { 901 struct instruction *insn; 902 struct section *sec; 903 struct symbol *func; 904 struct reloc *reloc; 905 906 sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard"); 907 if (!sec) 908 return; 909 910 list_for_each_entry(reloc, &sec->reloc_list, list) { 911 switch (reloc->sym->type) { 912 case STT_FUNC: 913 func = reloc->sym; 914 break; 915 916 case STT_SECTION: 917 func = find_func_by_offset(reloc->sym->sec, reloc->addend); 918 if (!func) 919 continue; 920 break; 921 922 default: 923 WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type); 924 continue; 925 } 926 927 func_for_each_insn(file, func, insn) 928 insn->ignore = true; 929 } 930 } 931 932 /* 933 * This is a whitelist of functions that is allowed to be called with AC set. 934 * The list is meant to be minimal and only contains compiler instrumentation 935 * ABI and a few functions used to implement *_{to,from}_user() functions. 936 * 937 * These functions must not directly change AC, but may PUSHF/POPF. 938 */ 939 static const char *uaccess_safe_builtin[] = { 940 /* KASAN */ 941 "kasan_report", 942 "kasan_check_range", 943 /* KASAN out-of-line */ 944 "__asan_loadN_noabort", 945 "__asan_load1_noabort", 946 "__asan_load2_noabort", 947 "__asan_load4_noabort", 948 "__asan_load8_noabort", 949 "__asan_load16_noabort", 950 "__asan_storeN_noabort", 951 "__asan_store1_noabort", 952 "__asan_store2_noabort", 953 "__asan_store4_noabort", 954 "__asan_store8_noabort", 955 "__asan_store16_noabort", 956 "__kasan_check_read", 957 "__kasan_check_write", 958 /* KASAN in-line */ 959 "__asan_report_load_n_noabort", 960 "__asan_report_load1_noabort", 961 "__asan_report_load2_noabort", 962 "__asan_report_load4_noabort", 963 "__asan_report_load8_noabort", 964 "__asan_report_load16_noabort", 965 "__asan_report_store_n_noabort", 966 "__asan_report_store1_noabort", 967 "__asan_report_store2_noabort", 968 "__asan_report_store4_noabort", 969 "__asan_report_store8_noabort", 970 "__asan_report_store16_noabort", 971 /* KCSAN */ 972 "__kcsan_check_access", 973 "__kcsan_mb", 974 "__kcsan_wmb", 975 "__kcsan_rmb", 976 "__kcsan_release", 977 "kcsan_found_watchpoint", 978 "kcsan_setup_watchpoint", 979 "kcsan_check_scoped_accesses", 980 "kcsan_disable_current", 981 "kcsan_enable_current_nowarn", 982 /* KCSAN/TSAN */ 983 "__tsan_func_entry", 984 "__tsan_func_exit", 985 "__tsan_read_range", 986 "__tsan_write_range", 987 "__tsan_read1", 988 "__tsan_read2", 989 "__tsan_read4", 990 "__tsan_read8", 991 "__tsan_read16", 992 "__tsan_write1", 993 "__tsan_write2", 994 "__tsan_write4", 995 "__tsan_write8", 996 "__tsan_write16", 997 "__tsan_read_write1", 998 "__tsan_read_write2", 999 "__tsan_read_write4", 1000 "__tsan_read_write8", 1001 "__tsan_read_write16", 1002 "__tsan_volatile_read1", 1003 "__tsan_volatile_read2", 1004 "__tsan_volatile_read4", 1005 "__tsan_volatile_read8", 1006 "__tsan_volatile_read16", 1007 "__tsan_volatile_write1", 1008 "__tsan_volatile_write2", 1009 "__tsan_volatile_write4", 1010 "__tsan_volatile_write8", 1011 "__tsan_volatile_write16", 1012 "__tsan_atomic8_load", 1013 "__tsan_atomic16_load", 1014 "__tsan_atomic32_load", 1015 "__tsan_atomic64_load", 1016 "__tsan_atomic8_store", 1017 "__tsan_atomic16_store", 1018 "__tsan_atomic32_store", 1019 "__tsan_atomic64_store", 1020 "__tsan_atomic8_exchange", 1021 "__tsan_atomic16_exchange", 1022 "__tsan_atomic32_exchange", 1023 "__tsan_atomic64_exchange", 1024 "__tsan_atomic8_fetch_add", 1025 "__tsan_atomic16_fetch_add", 1026 "__tsan_atomic32_fetch_add", 1027 "__tsan_atomic64_fetch_add", 1028 "__tsan_atomic8_fetch_sub", 1029 "__tsan_atomic16_fetch_sub", 1030 "__tsan_atomic32_fetch_sub", 1031 "__tsan_atomic64_fetch_sub", 1032 "__tsan_atomic8_fetch_and", 1033 "__tsan_atomic16_fetch_and", 1034 "__tsan_atomic32_fetch_and", 1035 "__tsan_atomic64_fetch_and", 1036 "__tsan_atomic8_fetch_or", 1037 "__tsan_atomic16_fetch_or", 1038 "__tsan_atomic32_fetch_or", 1039 "__tsan_atomic64_fetch_or", 1040 "__tsan_atomic8_fetch_xor", 1041 "__tsan_atomic16_fetch_xor", 1042 "__tsan_atomic32_fetch_xor", 1043 "__tsan_atomic64_fetch_xor", 1044 "__tsan_atomic8_fetch_nand", 1045 "__tsan_atomic16_fetch_nand", 1046 "__tsan_atomic32_fetch_nand", 1047 "__tsan_atomic64_fetch_nand", 1048 "__tsan_atomic8_compare_exchange_strong", 1049 "__tsan_atomic16_compare_exchange_strong", 1050 "__tsan_atomic32_compare_exchange_strong", 1051 "__tsan_atomic64_compare_exchange_strong", 1052 "__tsan_atomic8_compare_exchange_weak", 1053 "__tsan_atomic16_compare_exchange_weak", 1054 "__tsan_atomic32_compare_exchange_weak", 1055 "__tsan_atomic64_compare_exchange_weak", 1056 "__tsan_atomic8_compare_exchange_val", 1057 "__tsan_atomic16_compare_exchange_val", 1058 "__tsan_atomic32_compare_exchange_val", 1059 "__tsan_atomic64_compare_exchange_val", 1060 "__tsan_atomic_thread_fence", 1061 "__tsan_atomic_signal_fence", 1062 /* KCOV */ 1063 "write_comp_data", 1064 "check_kcov_mode", 1065 "__sanitizer_cov_trace_pc", 1066 "__sanitizer_cov_trace_const_cmp1", 1067 "__sanitizer_cov_trace_const_cmp2", 1068 "__sanitizer_cov_trace_const_cmp4", 1069 "__sanitizer_cov_trace_const_cmp8", 1070 "__sanitizer_cov_trace_cmp1", 1071 "__sanitizer_cov_trace_cmp2", 1072 "__sanitizer_cov_trace_cmp4", 1073 "__sanitizer_cov_trace_cmp8", 1074 "__sanitizer_cov_trace_switch", 1075 /* KMSAN */ 1076 "kmsan_copy_to_user", 1077 "kmsan_report", 1078 "kmsan_unpoison_entry_regs", 1079 "kmsan_unpoison_memory", 1080 "__msan_chain_origin", 1081 "__msan_get_context_state", 1082 "__msan_instrument_asm_store", 1083 "__msan_metadata_ptr_for_load_1", 1084 "__msan_metadata_ptr_for_load_2", 1085 "__msan_metadata_ptr_for_load_4", 1086 "__msan_metadata_ptr_for_load_8", 1087 "__msan_metadata_ptr_for_load_n", 1088 "__msan_metadata_ptr_for_store_1", 1089 "__msan_metadata_ptr_for_store_2", 1090 "__msan_metadata_ptr_for_store_4", 1091 "__msan_metadata_ptr_for_store_8", 1092 "__msan_metadata_ptr_for_store_n", 1093 "__msan_poison_alloca", 1094 "__msan_warning", 1095 /* UBSAN */ 1096 "ubsan_type_mismatch_common", 1097 "__ubsan_handle_type_mismatch", 1098 "__ubsan_handle_type_mismatch_v1", 1099 "__ubsan_handle_shift_out_of_bounds", 1100 /* misc */ 1101 "csum_partial_copy_generic", 1102 "copy_mc_fragile", 1103 "copy_mc_fragile_handle_tail", 1104 "copy_mc_enhanced_fast_string", 1105 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */ 1106 "clear_user_erms", 1107 "clear_user_rep_good", 1108 "clear_user_original", 1109 NULL 1110 }; 1111 1112 static void add_uaccess_safe(struct objtool_file *file) 1113 { 1114 struct symbol *func; 1115 const char **name; 1116 1117 if (!opts.uaccess) 1118 return; 1119 1120 for (name = uaccess_safe_builtin; *name; name++) { 1121 func = find_symbol_by_name(file->elf, *name); 1122 if (!func) 1123 continue; 1124 1125 func->uaccess_safe = true; 1126 } 1127 } 1128 1129 /* 1130 * FIXME: For now, just ignore any alternatives which add retpolines. This is 1131 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline. 1132 * But it at least allows objtool to understand the control flow *around* the 1133 * retpoline. 1134 */ 1135 static int add_ignore_alternatives(struct objtool_file *file) 1136 { 1137 struct section *sec; 1138 struct reloc *reloc; 1139 struct instruction *insn; 1140 1141 sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts"); 1142 if (!sec) 1143 return 0; 1144 1145 list_for_each_entry(reloc, &sec->reloc_list, list) { 1146 if (reloc->sym->type != STT_SECTION) { 1147 WARN("unexpected relocation symbol type in %s", sec->name); 1148 return -1; 1149 } 1150 1151 insn = find_insn(file, reloc->sym->sec, reloc->addend); 1152 if (!insn) { 1153 WARN("bad .discard.ignore_alts entry"); 1154 return -1; 1155 } 1156 1157 insn->ignore_alts = true; 1158 } 1159 1160 return 0; 1161 } 1162 1163 __weak bool arch_is_retpoline(struct symbol *sym) 1164 { 1165 return false; 1166 } 1167 1168 __weak bool arch_is_rethunk(struct symbol *sym) 1169 { 1170 return false; 1171 } 1172 1173 #define NEGATIVE_RELOC ((void *)-1L) 1174 1175 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) 1176 { 1177 if (insn->reloc == NEGATIVE_RELOC) 1178 return NULL; 1179 1180 if (!insn->reloc) { 1181 if (!file) 1182 return NULL; 1183 1184 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec, 1185 insn->offset, insn->len); 1186 if (!insn->reloc) { 1187 insn->reloc = NEGATIVE_RELOC; 1188 return NULL; 1189 } 1190 } 1191 1192 return insn->reloc; 1193 } 1194 1195 static void remove_insn_ops(struct instruction *insn) 1196 { 1197 struct stack_op *op, *tmp; 1198 1199 list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) { 1200 list_del(&op->list); 1201 free(op); 1202 } 1203 } 1204 1205 static void annotate_call_site(struct objtool_file *file, 1206 struct instruction *insn, bool sibling) 1207 { 1208 struct reloc *reloc = insn_reloc(file, insn); 1209 struct symbol *sym = insn->call_dest; 1210 1211 if (!sym) 1212 sym = reloc->sym; 1213 1214 /* 1215 * Alternative replacement code is just template code which is 1216 * sometimes copied to the original instruction. For now, don't 1217 * annotate it. (In the future we might consider annotating the 1218 * original instruction if/when it ever makes sense to do so.) 1219 */ 1220 if (!strcmp(insn->sec->name, ".altinstr_replacement")) 1221 return; 1222 1223 if (sym->static_call_tramp) { 1224 list_add_tail(&insn->call_node, &file->static_call_list); 1225 return; 1226 } 1227 1228 if (sym->retpoline_thunk) { 1229 list_add_tail(&insn->call_node, &file->retpoline_call_list); 1230 return; 1231 } 1232 1233 /* 1234 * Many compilers cannot disable KCOV or sanitizer calls with a function 1235 * attribute so they need a little help, NOP out any such calls from 1236 * noinstr text. 1237 */ 1238 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) { 1239 if (reloc) { 1240 reloc->type = R_NONE; 1241 elf_write_reloc(file->elf, reloc); 1242 } 1243 1244 elf_write_insn(file->elf, insn->sec, 1245 insn->offset, insn->len, 1246 sibling ? arch_ret_insn(insn->len) 1247 : arch_nop_insn(insn->len)); 1248 1249 insn->type = sibling ? INSN_RETURN : INSN_NOP; 1250 1251 if (sibling) { 1252 /* 1253 * We've replaced the tail-call JMP insn by two new 1254 * insn: RET; INT3, except we only have a single struct 1255 * insn here. Mark it retpoline_safe to avoid the SLS 1256 * warning, instead of adding another insn. 1257 */ 1258 insn->retpoline_safe = true; 1259 } 1260 1261 return; 1262 } 1263 1264 if (opts.mcount && sym->fentry) { 1265 if (sibling) 1266 WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset); 1267 1268 if (reloc) { 1269 reloc->type = R_NONE; 1270 elf_write_reloc(file->elf, reloc); 1271 } 1272 1273 elf_write_insn(file->elf, insn->sec, 1274 insn->offset, insn->len, 1275 arch_nop_insn(insn->len)); 1276 1277 insn->type = INSN_NOP; 1278 1279 list_add_tail(&insn->call_node, &file->mcount_loc_list); 1280 return; 1281 } 1282 1283 if (!sibling && dead_end_function(file, sym)) 1284 insn->dead_end = true; 1285 } 1286 1287 static void add_call_dest(struct objtool_file *file, struct instruction *insn, 1288 struct symbol *dest, bool sibling) 1289 { 1290 insn->call_dest = dest; 1291 if (!dest) 1292 return; 1293 1294 /* 1295 * Whatever stack impact regular CALLs have, should be undone 1296 * by the RETURN of the called function. 1297 * 1298 * Annotated intra-function calls retain the stack_ops but 1299 * are converted to JUMP, see read_intra_function_calls(). 1300 */ 1301 remove_insn_ops(insn); 1302 1303 annotate_call_site(file, insn, sibling); 1304 } 1305 1306 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn) 1307 { 1308 /* 1309 * Retpoline calls/jumps are really dynamic calls/jumps in disguise, 1310 * so convert them accordingly. 1311 */ 1312 switch (insn->type) { 1313 case INSN_CALL: 1314 insn->type = INSN_CALL_DYNAMIC; 1315 break; 1316 case INSN_JUMP_UNCONDITIONAL: 1317 insn->type = INSN_JUMP_DYNAMIC; 1318 break; 1319 case INSN_JUMP_CONDITIONAL: 1320 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL; 1321 break; 1322 default: 1323 return; 1324 } 1325 1326 insn->retpoline_safe = true; 1327 1328 /* 1329 * Whatever stack impact regular CALLs have, should be undone 1330 * by the RETURN of the called function. 1331 * 1332 * Annotated intra-function calls retain the stack_ops but 1333 * are converted to JUMP, see read_intra_function_calls(). 1334 */ 1335 remove_insn_ops(insn); 1336 1337 annotate_call_site(file, insn, false); 1338 } 1339 1340 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) 1341 { 1342 /* 1343 * Return thunk tail calls are really just returns in disguise, 1344 * so convert them accordingly. 1345 */ 1346 insn->type = INSN_RETURN; 1347 insn->retpoline_safe = true; 1348 1349 if (add) 1350 list_add_tail(&insn->call_node, &file->return_thunk_list); 1351 } 1352 1353 static bool same_function(struct instruction *insn1, struct instruction *insn2) 1354 { 1355 return insn1->func->pfunc == insn2->func->pfunc; 1356 } 1357 1358 static bool is_first_func_insn(struct objtool_file *file, struct instruction *insn) 1359 { 1360 if (insn->offset == insn->func->offset) 1361 return true; 1362 1363 if (opts.ibt) { 1364 struct instruction *prev = prev_insn_same_sym(file, insn); 1365 1366 if (prev && prev->type == INSN_ENDBR && 1367 insn->offset == insn->func->offset + prev->len) 1368 return true; 1369 } 1370 1371 return false; 1372 } 1373 1374 /* 1375 * Find the destination instructions for all jumps. 1376 */ 1377 static int add_jump_destinations(struct objtool_file *file) 1378 { 1379 struct instruction *insn, *jump_dest; 1380 struct reloc *reloc; 1381 struct section *dest_sec; 1382 unsigned long dest_off; 1383 1384 for_each_insn(file, insn) { 1385 if (insn->jump_dest) { 1386 /* 1387 * handle_group_alt() may have previously set 1388 * 'jump_dest' for some alternatives. 1389 */ 1390 continue; 1391 } 1392 if (!is_static_jump(insn)) 1393 continue; 1394 1395 reloc = insn_reloc(file, insn); 1396 if (!reloc) { 1397 dest_sec = insn->sec; 1398 dest_off = arch_jump_destination(insn); 1399 } else if (reloc->sym->type == STT_SECTION) { 1400 dest_sec = reloc->sym->sec; 1401 dest_off = arch_dest_reloc_offset(reloc->addend); 1402 } else if (reloc->sym->retpoline_thunk) { 1403 add_retpoline_call(file, insn); 1404 continue; 1405 } else if (reloc->sym->return_thunk) { 1406 add_return_call(file, insn, true); 1407 continue; 1408 } else if (insn->func) { 1409 /* 1410 * External sibling call or internal sibling call with 1411 * STT_FUNC reloc. 1412 */ 1413 add_call_dest(file, insn, reloc->sym, true); 1414 continue; 1415 } else if (reloc->sym->sec->idx) { 1416 dest_sec = reloc->sym->sec; 1417 dest_off = reloc->sym->sym.st_value + 1418 arch_dest_reloc_offset(reloc->addend); 1419 } else { 1420 /* non-func asm code jumping to another file */ 1421 continue; 1422 } 1423 1424 jump_dest = find_insn(file, dest_sec, dest_off); 1425 if (!jump_dest) { 1426 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); 1427 1428 /* 1429 * This is a special case for zen_untrain_ret(). 1430 * It jumps to __x86_return_thunk(), but objtool 1431 * can't find the thunk's starting RET 1432 * instruction, because the RET is also in the 1433 * middle of another instruction. Objtool only 1434 * knows about the outer instruction. 1435 */ 1436 if (sym && sym->return_thunk) { 1437 add_return_call(file, insn, false); 1438 continue; 1439 } 1440 1441 WARN_FUNC("can't find jump dest instruction at %s+0x%lx", 1442 insn->sec, insn->offset, dest_sec->name, 1443 dest_off); 1444 return -1; 1445 } 1446 1447 /* 1448 * Cross-function jump. 1449 */ 1450 if (insn->func && jump_dest->func && 1451 insn->func != jump_dest->func) { 1452 1453 /* 1454 * For GCC 8+, create parent/child links for any cold 1455 * subfunctions. This is _mostly_ redundant with a 1456 * similar initialization in read_symbols(). 1457 * 1458 * If a function has aliases, we want the *first* such 1459 * function in the symbol table to be the subfunction's 1460 * parent. In that case we overwrite the 1461 * initialization done in read_symbols(). 1462 * 1463 * However this code can't completely replace the 1464 * read_symbols() code because this doesn't detect the 1465 * case where the parent function's only reference to a 1466 * subfunction is through a jump table. 1467 */ 1468 if (!strstr(insn->func->name, ".cold") && 1469 strstr(jump_dest->func->name, ".cold")) { 1470 insn->func->cfunc = jump_dest->func; 1471 jump_dest->func->pfunc = insn->func; 1472 1473 } else if (!same_function(insn, jump_dest) && 1474 is_first_func_insn(file, jump_dest)) { 1475 /* 1476 * Internal sibling call without reloc or with 1477 * STT_SECTION reloc. 1478 */ 1479 add_call_dest(file, insn, jump_dest->func, true); 1480 continue; 1481 } 1482 } 1483 1484 insn->jump_dest = jump_dest; 1485 } 1486 1487 return 0; 1488 } 1489 1490 static struct symbol *find_call_destination(struct section *sec, unsigned long offset) 1491 { 1492 struct symbol *call_dest; 1493 1494 call_dest = find_func_by_offset(sec, offset); 1495 if (!call_dest) 1496 call_dest = find_symbol_by_offset(sec, offset); 1497 1498 return call_dest; 1499 } 1500 1501 /* 1502 * Find the destination instructions for all calls. 1503 */ 1504 static int add_call_destinations(struct objtool_file *file) 1505 { 1506 struct instruction *insn; 1507 unsigned long dest_off; 1508 struct symbol *dest; 1509 struct reloc *reloc; 1510 1511 for_each_insn(file, insn) { 1512 if (insn->type != INSN_CALL) 1513 continue; 1514 1515 reloc = insn_reloc(file, insn); 1516 if (!reloc) { 1517 dest_off = arch_jump_destination(insn); 1518 dest = find_call_destination(insn->sec, dest_off); 1519 1520 add_call_dest(file, insn, dest, false); 1521 1522 if (insn->ignore) 1523 continue; 1524 1525 if (!insn->call_dest) { 1526 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset); 1527 return -1; 1528 } 1529 1530 if (insn->func && insn->call_dest->type != STT_FUNC) { 1531 WARN_FUNC("unsupported call to non-function", 1532 insn->sec, insn->offset); 1533 return -1; 1534 } 1535 1536 } else if (reloc->sym->type == STT_SECTION) { 1537 dest_off = arch_dest_reloc_offset(reloc->addend); 1538 dest = find_call_destination(reloc->sym->sec, dest_off); 1539 if (!dest) { 1540 WARN_FUNC("can't find call dest symbol at %s+0x%lx", 1541 insn->sec, insn->offset, 1542 reloc->sym->sec->name, 1543 dest_off); 1544 return -1; 1545 } 1546 1547 add_call_dest(file, insn, dest, false); 1548 1549 } else if (reloc->sym->retpoline_thunk) { 1550 add_retpoline_call(file, insn); 1551 1552 } else 1553 add_call_dest(file, insn, reloc->sym, false); 1554 } 1555 1556 return 0; 1557 } 1558 1559 /* 1560 * The .alternatives section requires some extra special care over and above 1561 * other special sections because alternatives are patched in place. 1562 */ 1563 static int handle_group_alt(struct objtool_file *file, 1564 struct special_alt *special_alt, 1565 struct instruction *orig_insn, 1566 struct instruction **new_insn) 1567 { 1568 struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL; 1569 struct alt_group *orig_alt_group, *new_alt_group; 1570 unsigned long dest_off; 1571 1572 1573 orig_alt_group = malloc(sizeof(*orig_alt_group)); 1574 if (!orig_alt_group) { 1575 WARN("malloc failed"); 1576 return -1; 1577 } 1578 orig_alt_group->cfi = calloc(special_alt->orig_len, 1579 sizeof(struct cfi_state *)); 1580 if (!orig_alt_group->cfi) { 1581 WARN("calloc failed"); 1582 return -1; 1583 } 1584 1585 last_orig_insn = NULL; 1586 insn = orig_insn; 1587 sec_for_each_insn_from(file, insn) { 1588 if (insn->offset >= special_alt->orig_off + special_alt->orig_len) 1589 break; 1590 1591 insn->alt_group = orig_alt_group; 1592 last_orig_insn = insn; 1593 } 1594 orig_alt_group->orig_group = NULL; 1595 orig_alt_group->first_insn = orig_insn; 1596 orig_alt_group->last_insn = last_orig_insn; 1597 1598 1599 new_alt_group = malloc(sizeof(*new_alt_group)); 1600 if (!new_alt_group) { 1601 WARN("malloc failed"); 1602 return -1; 1603 } 1604 1605 if (special_alt->new_len < special_alt->orig_len) { 1606 /* 1607 * Insert a fake nop at the end to make the replacement 1608 * alt_group the same size as the original. This is needed to 1609 * allow propagate_alt_cfi() to do its magic. When the last 1610 * instruction affects the stack, the instruction after it (the 1611 * nop) will propagate the new state to the shared CFI array. 1612 */ 1613 nop = malloc(sizeof(*nop)); 1614 if (!nop) { 1615 WARN("malloc failed"); 1616 return -1; 1617 } 1618 memset(nop, 0, sizeof(*nop)); 1619 INIT_LIST_HEAD(&nop->alts); 1620 INIT_LIST_HEAD(&nop->stack_ops); 1621 1622 nop->sec = special_alt->new_sec; 1623 nop->offset = special_alt->new_off + special_alt->new_len; 1624 nop->len = special_alt->orig_len - special_alt->new_len; 1625 nop->type = INSN_NOP; 1626 nop->func = orig_insn->func; 1627 nop->alt_group = new_alt_group; 1628 nop->ignore = orig_insn->ignore_alts; 1629 } 1630 1631 if (!special_alt->new_len) { 1632 *new_insn = nop; 1633 goto end; 1634 } 1635 1636 insn = *new_insn; 1637 sec_for_each_insn_from(file, insn) { 1638 struct reloc *alt_reloc; 1639 1640 if (insn->offset >= special_alt->new_off + special_alt->new_len) 1641 break; 1642 1643 last_new_insn = insn; 1644 1645 insn->ignore = orig_insn->ignore_alts; 1646 insn->func = orig_insn->func; 1647 insn->alt_group = new_alt_group; 1648 1649 /* 1650 * Since alternative replacement code is copy/pasted by the 1651 * kernel after applying relocations, generally such code can't 1652 * have relative-address relocation references to outside the 1653 * .altinstr_replacement section, unless the arch's 1654 * alternatives code can adjust the relative offsets 1655 * accordingly. 1656 */ 1657 alt_reloc = insn_reloc(file, insn); 1658 if (alt_reloc && 1659 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { 1660 1661 WARN_FUNC("unsupported relocation in alternatives section", 1662 insn->sec, insn->offset); 1663 return -1; 1664 } 1665 1666 if (!is_static_jump(insn)) 1667 continue; 1668 1669 if (!insn->immediate) 1670 continue; 1671 1672 dest_off = arch_jump_destination(insn); 1673 if (dest_off == special_alt->new_off + special_alt->new_len) { 1674 insn->jump_dest = next_insn_same_sec(file, last_orig_insn); 1675 if (!insn->jump_dest) { 1676 WARN_FUNC("can't find alternative jump destination", 1677 insn->sec, insn->offset); 1678 return -1; 1679 } 1680 } 1681 } 1682 1683 if (!last_new_insn) { 1684 WARN_FUNC("can't find last new alternative instruction", 1685 special_alt->new_sec, special_alt->new_off); 1686 return -1; 1687 } 1688 1689 if (nop) 1690 list_add(&nop->list, &last_new_insn->list); 1691 end: 1692 new_alt_group->orig_group = orig_alt_group; 1693 new_alt_group->first_insn = *new_insn; 1694 new_alt_group->last_insn = nop ? : last_new_insn; 1695 new_alt_group->cfi = orig_alt_group->cfi; 1696 return 0; 1697 } 1698 1699 /* 1700 * A jump table entry can either convert a nop to a jump or a jump to a nop. 1701 * If the original instruction is a jump, make the alt entry an effective nop 1702 * by just skipping the original instruction. 1703 */ 1704 static int handle_jump_alt(struct objtool_file *file, 1705 struct special_alt *special_alt, 1706 struct instruction *orig_insn, 1707 struct instruction **new_insn) 1708 { 1709 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && 1710 orig_insn->type != INSN_NOP) { 1711 1712 WARN_FUNC("unsupported instruction at jump label", 1713 orig_insn->sec, orig_insn->offset); 1714 return -1; 1715 } 1716 1717 if (opts.hack_jump_label && special_alt->key_addend & 2) { 1718 struct reloc *reloc = insn_reloc(file, orig_insn); 1719 1720 if (reloc) { 1721 reloc->type = R_NONE; 1722 elf_write_reloc(file->elf, reloc); 1723 } 1724 elf_write_insn(file->elf, orig_insn->sec, 1725 orig_insn->offset, orig_insn->len, 1726 arch_nop_insn(orig_insn->len)); 1727 orig_insn->type = INSN_NOP; 1728 } 1729 1730 if (orig_insn->type == INSN_NOP) { 1731 if (orig_insn->len == 2) 1732 file->jl_nop_short++; 1733 else 1734 file->jl_nop_long++; 1735 1736 return 0; 1737 } 1738 1739 if (orig_insn->len == 2) 1740 file->jl_short++; 1741 else 1742 file->jl_long++; 1743 1744 *new_insn = list_next_entry(orig_insn, list); 1745 return 0; 1746 } 1747 1748 /* 1749 * Read all the special sections which have alternate instructions which can be 1750 * patched in or redirected to at runtime. Each instruction having alternate 1751 * instruction(s) has them added to its insn->alts list, which will be 1752 * traversed in validate_branch(). 1753 */ 1754 static int add_special_section_alts(struct objtool_file *file) 1755 { 1756 struct list_head special_alts; 1757 struct instruction *orig_insn, *new_insn; 1758 struct special_alt *special_alt, *tmp; 1759 struct alternative *alt; 1760 int ret; 1761 1762 ret = special_get_alts(file->elf, &special_alts); 1763 if (ret) 1764 return ret; 1765 1766 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) { 1767 1768 orig_insn = find_insn(file, special_alt->orig_sec, 1769 special_alt->orig_off); 1770 if (!orig_insn) { 1771 WARN_FUNC("special: can't find orig instruction", 1772 special_alt->orig_sec, special_alt->orig_off); 1773 ret = -1; 1774 goto out; 1775 } 1776 1777 new_insn = NULL; 1778 if (!special_alt->group || special_alt->new_len) { 1779 new_insn = find_insn(file, special_alt->new_sec, 1780 special_alt->new_off); 1781 if (!new_insn) { 1782 WARN_FUNC("special: can't find new instruction", 1783 special_alt->new_sec, 1784 special_alt->new_off); 1785 ret = -1; 1786 goto out; 1787 } 1788 } 1789 1790 if (special_alt->group) { 1791 if (!special_alt->orig_len) { 1792 WARN_FUNC("empty alternative entry", 1793 orig_insn->sec, orig_insn->offset); 1794 continue; 1795 } 1796 1797 ret = handle_group_alt(file, special_alt, orig_insn, 1798 &new_insn); 1799 if (ret) 1800 goto out; 1801 } else if (special_alt->jump_or_nop) { 1802 ret = handle_jump_alt(file, special_alt, orig_insn, 1803 &new_insn); 1804 if (ret) 1805 goto out; 1806 } 1807 1808 alt = malloc(sizeof(*alt)); 1809 if (!alt) { 1810 WARN("malloc failed"); 1811 ret = -1; 1812 goto out; 1813 } 1814 1815 alt->insn = new_insn; 1816 alt->skip_orig = special_alt->skip_orig; 1817 orig_insn->ignore_alts |= special_alt->skip_alt; 1818 list_add_tail(&alt->list, &orig_insn->alts); 1819 1820 list_del(&special_alt->list); 1821 free(special_alt); 1822 } 1823 1824 if (opts.stats) { 1825 printf("jl\\\tNOP\tJMP\n"); 1826 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); 1827 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); 1828 } 1829 1830 out: 1831 return ret; 1832 } 1833 1834 static int add_jump_table(struct objtool_file *file, struct instruction *insn, 1835 struct reloc *table) 1836 { 1837 struct reloc *reloc = table; 1838 struct instruction *dest_insn; 1839 struct alternative *alt; 1840 struct symbol *pfunc = insn->func->pfunc; 1841 unsigned int prev_offset = 0; 1842 1843 /* 1844 * Each @reloc is a switch table relocation which points to the target 1845 * instruction. 1846 */ 1847 list_for_each_entry_from(reloc, &table->sec->reloc_list, list) { 1848 1849 /* Check for the end of the table: */ 1850 if (reloc != table && reloc->jump_table_start) 1851 break; 1852 1853 /* Make sure the table entries are consecutive: */ 1854 if (prev_offset && reloc->offset != prev_offset + 8) 1855 break; 1856 1857 /* Detect function pointers from contiguous objects: */ 1858 if (reloc->sym->sec == pfunc->sec && 1859 reloc->addend == pfunc->offset) 1860 break; 1861 1862 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend); 1863 if (!dest_insn) 1864 break; 1865 1866 /* Make sure the destination is in the same function: */ 1867 if (!dest_insn->func || dest_insn->func->pfunc != pfunc) 1868 break; 1869 1870 alt = malloc(sizeof(*alt)); 1871 if (!alt) { 1872 WARN("malloc failed"); 1873 return -1; 1874 } 1875 1876 alt->insn = dest_insn; 1877 list_add_tail(&alt->list, &insn->alts); 1878 prev_offset = reloc->offset; 1879 } 1880 1881 if (!prev_offset) { 1882 WARN_FUNC("can't find switch jump table", 1883 insn->sec, insn->offset); 1884 return -1; 1885 } 1886 1887 return 0; 1888 } 1889 1890 /* 1891 * find_jump_table() - Given a dynamic jump, find the switch jump table 1892 * associated with it. 1893 */ 1894 static struct reloc *find_jump_table(struct objtool_file *file, 1895 struct symbol *func, 1896 struct instruction *insn) 1897 { 1898 struct reloc *table_reloc; 1899 struct instruction *dest_insn, *orig_insn = insn; 1900 1901 /* 1902 * Backward search using the @first_jump_src links, these help avoid 1903 * much of the 'in between' code. Which avoids us getting confused by 1904 * it. 1905 */ 1906 for (; 1907 insn && insn->func && insn->func->pfunc == func; 1908 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) { 1909 1910 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC) 1911 break; 1912 1913 /* allow small jumps within the range */ 1914 if (insn->type == INSN_JUMP_UNCONDITIONAL && 1915 insn->jump_dest && 1916 (insn->jump_dest->offset <= insn->offset || 1917 insn->jump_dest->offset > orig_insn->offset)) 1918 break; 1919 1920 table_reloc = arch_find_switch_table(file, insn); 1921 if (!table_reloc) 1922 continue; 1923 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend); 1924 if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func) 1925 continue; 1926 1927 return table_reloc; 1928 } 1929 1930 return NULL; 1931 } 1932 1933 /* 1934 * First pass: Mark the head of each jump table so that in the next pass, 1935 * we know when a given jump table ends and the next one starts. 1936 */ 1937 static void mark_func_jump_tables(struct objtool_file *file, 1938 struct symbol *func) 1939 { 1940 struct instruction *insn, *last = NULL; 1941 struct reloc *reloc; 1942 1943 func_for_each_insn(file, func, insn) { 1944 if (!last) 1945 last = insn; 1946 1947 /* 1948 * Store back-pointers for unconditional forward jumps such 1949 * that find_jump_table() can back-track using those and 1950 * avoid some potentially confusing code. 1951 */ 1952 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && 1953 insn->offset > last->offset && 1954 insn->jump_dest->offset > insn->offset && 1955 !insn->jump_dest->first_jump_src) { 1956 1957 insn->jump_dest->first_jump_src = insn; 1958 last = insn->jump_dest; 1959 } 1960 1961 if (insn->type != INSN_JUMP_DYNAMIC) 1962 continue; 1963 1964 reloc = find_jump_table(file, func, insn); 1965 if (reloc) { 1966 reloc->jump_table_start = true; 1967 insn->jump_table = reloc; 1968 } 1969 } 1970 } 1971 1972 static int add_func_jump_tables(struct objtool_file *file, 1973 struct symbol *func) 1974 { 1975 struct instruction *insn; 1976 int ret; 1977 1978 func_for_each_insn(file, func, insn) { 1979 if (!insn->jump_table) 1980 continue; 1981 1982 ret = add_jump_table(file, insn, insn->jump_table); 1983 if (ret) 1984 return ret; 1985 } 1986 1987 return 0; 1988 } 1989 1990 /* 1991 * For some switch statements, gcc generates a jump table in the .rodata 1992 * section which contains a list of addresses within the function to jump to. 1993 * This finds these jump tables and adds them to the insn->alts lists. 1994 */ 1995 static int add_jump_table_alts(struct objtool_file *file) 1996 { 1997 struct section *sec; 1998 struct symbol *func; 1999 int ret; 2000 2001 if (!file->rodata) 2002 return 0; 2003 2004 for_each_sec(file, sec) { 2005 list_for_each_entry(func, &sec->symbol_list, list) { 2006 if (func->type != STT_FUNC) 2007 continue; 2008 2009 mark_func_jump_tables(file, func); 2010 ret = add_func_jump_tables(file, func); 2011 if (ret) 2012 return ret; 2013 } 2014 } 2015 2016 return 0; 2017 } 2018 2019 static void set_func_state(struct cfi_state *state) 2020 { 2021 state->cfa = initial_func_cfi.cfa; 2022 memcpy(&state->regs, &initial_func_cfi.regs, 2023 CFI_NUM_REGS * sizeof(struct cfi_reg)); 2024 state->stack_size = initial_func_cfi.cfa.offset; 2025 } 2026 2027 static int read_unwind_hints(struct objtool_file *file) 2028 { 2029 struct cfi_state cfi = init_cfi; 2030 struct section *sec, *relocsec; 2031 struct unwind_hint *hint; 2032 struct instruction *insn; 2033 struct reloc *reloc; 2034 int i; 2035 2036 sec = find_section_by_name(file->elf, ".discard.unwind_hints"); 2037 if (!sec) 2038 return 0; 2039 2040 relocsec = sec->reloc; 2041 if (!relocsec) { 2042 WARN("missing .rela.discard.unwind_hints section"); 2043 return -1; 2044 } 2045 2046 if (sec->sh.sh_size % sizeof(struct unwind_hint)) { 2047 WARN("struct unwind_hint size mismatch"); 2048 return -1; 2049 } 2050 2051 file->hints = true; 2052 2053 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) { 2054 hint = (struct unwind_hint *)sec->data->d_buf + i; 2055 2056 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); 2057 if (!reloc) { 2058 WARN("can't find reloc for unwind_hints[%d]", i); 2059 return -1; 2060 } 2061 2062 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2063 if (!insn) { 2064 WARN("can't find insn for unwind_hints[%d]", i); 2065 return -1; 2066 } 2067 2068 insn->hint = true; 2069 2070 if (hint->type == UNWIND_HINT_TYPE_SAVE) { 2071 insn->hint = false; 2072 insn->save = true; 2073 continue; 2074 } 2075 2076 if (hint->type == UNWIND_HINT_TYPE_RESTORE) { 2077 insn->restore = true; 2078 continue; 2079 } 2080 2081 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 2082 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 2083 2084 if (sym && sym->bind == STB_GLOBAL) { 2085 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { 2086 WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR", 2087 insn->sec, insn->offset); 2088 } 2089 2090 insn->entry = 1; 2091 } 2092 } 2093 2094 if (hint->type == UNWIND_HINT_TYPE_ENTRY) { 2095 hint->type = UNWIND_HINT_TYPE_CALL; 2096 insn->entry = 1; 2097 } 2098 2099 if (hint->type == UNWIND_HINT_TYPE_FUNC) { 2100 insn->cfi = &func_cfi; 2101 continue; 2102 } 2103 2104 if (insn->cfi) 2105 cfi = *(insn->cfi); 2106 2107 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { 2108 WARN_FUNC("unsupported unwind_hint sp base reg %d", 2109 insn->sec, insn->offset, hint->sp_reg); 2110 return -1; 2111 } 2112 2113 cfi.cfa.offset = bswap_if_needed(hint->sp_offset); 2114 cfi.type = hint->type; 2115 cfi.end = hint->end; 2116 2117 insn->cfi = cfi_hash_find_or_add(&cfi); 2118 } 2119 2120 return 0; 2121 } 2122 2123 static int read_noendbr_hints(struct objtool_file *file) 2124 { 2125 struct section *sec; 2126 struct instruction *insn; 2127 struct reloc *reloc; 2128 2129 sec = find_section_by_name(file->elf, ".rela.discard.noendbr"); 2130 if (!sec) 2131 return 0; 2132 2133 list_for_each_entry(reloc, &sec->reloc_list, list) { 2134 insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend); 2135 if (!insn) { 2136 WARN("bad .discard.noendbr entry"); 2137 return -1; 2138 } 2139 2140 insn->noendbr = 1; 2141 } 2142 2143 return 0; 2144 } 2145 2146 static int read_retpoline_hints(struct objtool_file *file) 2147 { 2148 struct section *sec; 2149 struct instruction *insn; 2150 struct reloc *reloc; 2151 2152 sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe"); 2153 if (!sec) 2154 return 0; 2155 2156 list_for_each_entry(reloc, &sec->reloc_list, list) { 2157 if (reloc->sym->type != STT_SECTION) { 2158 WARN("unexpected relocation symbol type in %s", sec->name); 2159 return -1; 2160 } 2161 2162 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2163 if (!insn) { 2164 WARN("bad .discard.retpoline_safe entry"); 2165 return -1; 2166 } 2167 2168 if (insn->type != INSN_JUMP_DYNAMIC && 2169 insn->type != INSN_CALL_DYNAMIC && 2170 insn->type != INSN_RETURN && 2171 insn->type != INSN_NOP) { 2172 WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop", 2173 insn->sec, insn->offset); 2174 return -1; 2175 } 2176 2177 insn->retpoline_safe = true; 2178 } 2179 2180 return 0; 2181 } 2182 2183 static int read_instr_hints(struct objtool_file *file) 2184 { 2185 struct section *sec; 2186 struct instruction *insn; 2187 struct reloc *reloc; 2188 2189 sec = find_section_by_name(file->elf, ".rela.discard.instr_end"); 2190 if (!sec) 2191 return 0; 2192 2193 list_for_each_entry(reloc, &sec->reloc_list, list) { 2194 if (reloc->sym->type != STT_SECTION) { 2195 WARN("unexpected relocation symbol type in %s", sec->name); 2196 return -1; 2197 } 2198 2199 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2200 if (!insn) { 2201 WARN("bad .discard.instr_end entry"); 2202 return -1; 2203 } 2204 2205 insn->instr--; 2206 } 2207 2208 sec = find_section_by_name(file->elf, ".rela.discard.instr_begin"); 2209 if (!sec) 2210 return 0; 2211 2212 list_for_each_entry(reloc, &sec->reloc_list, list) { 2213 if (reloc->sym->type != STT_SECTION) { 2214 WARN("unexpected relocation symbol type in %s", sec->name); 2215 return -1; 2216 } 2217 2218 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2219 if (!insn) { 2220 WARN("bad .discard.instr_begin entry"); 2221 return -1; 2222 } 2223 2224 insn->instr++; 2225 } 2226 2227 return 0; 2228 } 2229 2230 static int read_intra_function_calls(struct objtool_file *file) 2231 { 2232 struct instruction *insn; 2233 struct section *sec; 2234 struct reloc *reloc; 2235 2236 sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls"); 2237 if (!sec) 2238 return 0; 2239 2240 list_for_each_entry(reloc, &sec->reloc_list, list) { 2241 unsigned long dest_off; 2242 2243 if (reloc->sym->type != STT_SECTION) { 2244 WARN("unexpected relocation symbol type in %s", 2245 sec->name); 2246 return -1; 2247 } 2248 2249 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2250 if (!insn) { 2251 WARN("bad .discard.intra_function_call entry"); 2252 return -1; 2253 } 2254 2255 if (insn->type != INSN_CALL) { 2256 WARN_FUNC("intra_function_call not a direct call", 2257 insn->sec, insn->offset); 2258 return -1; 2259 } 2260 2261 /* 2262 * Treat intra-function CALLs as JMPs, but with a stack_op. 2263 * See add_call_destinations(), which strips stack_ops from 2264 * normal CALLs. 2265 */ 2266 insn->type = INSN_JUMP_UNCONDITIONAL; 2267 2268 dest_off = arch_jump_destination(insn); 2269 insn->jump_dest = find_insn(file, insn->sec, dest_off); 2270 if (!insn->jump_dest) { 2271 WARN_FUNC("can't find call dest at %s+0x%lx", 2272 insn->sec, insn->offset, 2273 insn->sec->name, dest_off); 2274 return -1; 2275 } 2276 } 2277 2278 return 0; 2279 } 2280 2281 /* 2282 * Return true if name matches an instrumentation function, where calls to that 2283 * function from noinstr code can safely be removed, but compilers won't do so. 2284 */ 2285 static bool is_profiling_func(const char *name) 2286 { 2287 /* 2288 * Many compilers cannot disable KCOV with a function attribute. 2289 */ 2290 if (!strncmp(name, "__sanitizer_cov_", 16)) 2291 return true; 2292 2293 /* 2294 * Some compilers currently do not remove __tsan_func_entry/exit nor 2295 * __tsan_atomic_signal_fence (used for barrier instrumentation) with 2296 * the __no_sanitize_thread attribute, remove them. Once the kernel's 2297 * minimum Clang version is 14.0, this can be removed. 2298 */ 2299 if (!strncmp(name, "__tsan_func_", 12) || 2300 !strcmp(name, "__tsan_atomic_signal_fence")) 2301 return true; 2302 2303 return false; 2304 } 2305 2306 static int classify_symbols(struct objtool_file *file) 2307 { 2308 struct section *sec; 2309 struct symbol *func; 2310 2311 for_each_sec(file, sec) { 2312 list_for_each_entry(func, &sec->symbol_list, list) { 2313 if (func->bind != STB_GLOBAL) 2314 continue; 2315 2316 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, 2317 strlen(STATIC_CALL_TRAMP_PREFIX_STR))) 2318 func->static_call_tramp = true; 2319 2320 if (arch_is_retpoline(func)) 2321 func->retpoline_thunk = true; 2322 2323 if (arch_is_rethunk(func)) 2324 func->return_thunk = true; 2325 2326 if (!strcmp(func->name, "__fentry__")) 2327 func->fentry = true; 2328 2329 if (is_profiling_func(func->name)) 2330 func->profiling_func = true; 2331 } 2332 } 2333 2334 return 0; 2335 } 2336 2337 static void mark_rodata(struct objtool_file *file) 2338 { 2339 struct section *sec; 2340 bool found = false; 2341 2342 /* 2343 * Search for the following rodata sections, each of which can 2344 * potentially contain jump tables: 2345 * 2346 * - .rodata: can contain GCC switch tables 2347 * - .rodata.<func>: same, if -fdata-sections is being used 2348 * - .rodata..c_jump_table: contains C annotated jump tables 2349 * 2350 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2351 */ 2352 for_each_sec(file, sec) { 2353 if (!strncmp(sec->name, ".rodata", 7) && 2354 !strstr(sec->name, ".str1.")) { 2355 sec->rodata = true; 2356 found = true; 2357 } 2358 } 2359 2360 file->rodata = found; 2361 } 2362 2363 static int decode_sections(struct objtool_file *file) 2364 { 2365 int ret; 2366 2367 mark_rodata(file); 2368 2369 ret = init_pv_ops(file); 2370 if (ret) 2371 return ret; 2372 2373 ret = decode_instructions(file); 2374 if (ret) 2375 return ret; 2376 2377 add_ignores(file); 2378 add_uaccess_safe(file); 2379 2380 ret = add_ignore_alternatives(file); 2381 if (ret) 2382 return ret; 2383 2384 /* 2385 * Must be before read_unwind_hints() since that needs insn->noendbr. 2386 */ 2387 ret = read_noendbr_hints(file); 2388 if (ret) 2389 return ret; 2390 2391 /* 2392 * Must be before add_{jump_call}_destination. 2393 */ 2394 ret = classify_symbols(file); 2395 if (ret) 2396 return ret; 2397 2398 /* 2399 * Must be before add_jump_destinations(), which depends on 'func' 2400 * being set for alternatives, to enable proper sibling call detection. 2401 */ 2402 ret = add_special_section_alts(file); 2403 if (ret) 2404 return ret; 2405 2406 ret = add_jump_destinations(file); 2407 if (ret) 2408 return ret; 2409 2410 /* 2411 * Must be before add_call_destination(); it changes INSN_CALL to 2412 * INSN_JUMP. 2413 */ 2414 ret = read_intra_function_calls(file); 2415 if (ret) 2416 return ret; 2417 2418 ret = add_call_destinations(file); 2419 if (ret) 2420 return ret; 2421 2422 /* 2423 * Must be after add_call_destinations() such that it can override 2424 * dead_end_function() marks. 2425 */ 2426 ret = add_dead_ends(file); 2427 if (ret) 2428 return ret; 2429 2430 ret = add_jump_table_alts(file); 2431 if (ret) 2432 return ret; 2433 2434 ret = read_unwind_hints(file); 2435 if (ret) 2436 return ret; 2437 2438 ret = read_retpoline_hints(file); 2439 if (ret) 2440 return ret; 2441 2442 ret = read_instr_hints(file); 2443 if (ret) 2444 return ret; 2445 2446 return 0; 2447 } 2448 2449 static bool is_fentry_call(struct instruction *insn) 2450 { 2451 if (insn->type == INSN_CALL && 2452 insn->call_dest && 2453 insn->call_dest->fentry) 2454 return true; 2455 2456 return false; 2457 } 2458 2459 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state) 2460 { 2461 struct cfi_state *cfi = &state->cfi; 2462 int i; 2463 2464 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap) 2465 return true; 2466 2467 if (cfi->cfa.offset != initial_func_cfi.cfa.offset) 2468 return true; 2469 2470 if (cfi->stack_size != initial_func_cfi.cfa.offset) 2471 return true; 2472 2473 for (i = 0; i < CFI_NUM_REGS; i++) { 2474 if (cfi->regs[i].base != initial_func_cfi.regs[i].base || 2475 cfi->regs[i].offset != initial_func_cfi.regs[i].offset) 2476 return true; 2477 } 2478 2479 return false; 2480 } 2481 2482 static bool check_reg_frame_pos(const struct cfi_reg *reg, 2483 int expected_offset) 2484 { 2485 return reg->base == CFI_CFA && 2486 reg->offset == expected_offset; 2487 } 2488 2489 static bool has_valid_stack_frame(struct insn_state *state) 2490 { 2491 struct cfi_state *cfi = &state->cfi; 2492 2493 if (cfi->cfa.base == CFI_BP && 2494 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) && 2495 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8)) 2496 return true; 2497 2498 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP) 2499 return true; 2500 2501 return false; 2502 } 2503 2504 static int update_cfi_state_regs(struct instruction *insn, 2505 struct cfi_state *cfi, 2506 struct stack_op *op) 2507 { 2508 struct cfi_reg *cfa = &cfi->cfa; 2509 2510 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT) 2511 return 0; 2512 2513 /* push */ 2514 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF) 2515 cfa->offset += 8; 2516 2517 /* pop */ 2518 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF) 2519 cfa->offset -= 8; 2520 2521 /* add immediate to sp */ 2522 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD && 2523 op->dest.reg == CFI_SP && op->src.reg == CFI_SP) 2524 cfa->offset -= op->src.offset; 2525 2526 return 0; 2527 } 2528 2529 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset) 2530 { 2531 if (arch_callee_saved_reg(reg) && 2532 cfi->regs[reg].base == CFI_UNDEFINED) { 2533 cfi->regs[reg].base = base; 2534 cfi->regs[reg].offset = offset; 2535 } 2536 } 2537 2538 static void restore_reg(struct cfi_state *cfi, unsigned char reg) 2539 { 2540 cfi->regs[reg].base = initial_func_cfi.regs[reg].base; 2541 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset; 2542 } 2543 2544 /* 2545 * A note about DRAP stack alignment: 2546 * 2547 * GCC has the concept of a DRAP register, which is used to help keep track of 2548 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP 2549 * register. The typical DRAP pattern is: 2550 * 2551 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10 2552 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp 2553 * 41 ff 72 f8 pushq -0x8(%r10) 2554 * 55 push %rbp 2555 * 48 89 e5 mov %rsp,%rbp 2556 * (more pushes) 2557 * 41 52 push %r10 2558 * ... 2559 * 41 5a pop %r10 2560 * (more pops) 2561 * 5d pop %rbp 2562 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2563 * c3 retq 2564 * 2565 * There are some variations in the epilogues, like: 2566 * 2567 * 5b pop %rbx 2568 * 41 5a pop %r10 2569 * 41 5c pop %r12 2570 * 41 5d pop %r13 2571 * 41 5e pop %r14 2572 * c9 leaveq 2573 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2574 * c3 retq 2575 * 2576 * and: 2577 * 2578 * 4c 8b 55 e8 mov -0x18(%rbp),%r10 2579 * 48 8b 5d e0 mov -0x20(%rbp),%rbx 2580 * 4c 8b 65 f0 mov -0x10(%rbp),%r12 2581 * 4c 8b 6d f8 mov -0x8(%rbp),%r13 2582 * c9 leaveq 2583 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2584 * c3 retq 2585 * 2586 * Sometimes r13 is used as the DRAP register, in which case it's saved and 2587 * restored beforehand: 2588 * 2589 * 41 55 push %r13 2590 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13 2591 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp 2592 * ... 2593 * 49 8d 65 f0 lea -0x10(%r13),%rsp 2594 * 41 5d pop %r13 2595 * c3 retq 2596 */ 2597 static int update_cfi_state(struct instruction *insn, 2598 struct instruction *next_insn, 2599 struct cfi_state *cfi, struct stack_op *op) 2600 { 2601 struct cfi_reg *cfa = &cfi->cfa; 2602 struct cfi_reg *regs = cfi->regs; 2603 2604 /* stack operations don't make sense with an undefined CFA */ 2605 if (cfa->base == CFI_UNDEFINED) { 2606 if (insn->func) { 2607 WARN_FUNC("undefined stack state", insn->sec, insn->offset); 2608 return -1; 2609 } 2610 return 0; 2611 } 2612 2613 if (cfi->type == UNWIND_HINT_TYPE_REGS || 2614 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) 2615 return update_cfi_state_regs(insn, cfi, op); 2616 2617 switch (op->dest.type) { 2618 2619 case OP_DEST_REG: 2620 switch (op->src.type) { 2621 2622 case OP_SRC_REG: 2623 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP && 2624 cfa->base == CFI_SP && 2625 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) { 2626 2627 /* mov %rsp, %rbp */ 2628 cfa->base = op->dest.reg; 2629 cfi->bp_scratch = false; 2630 } 2631 2632 else if (op->src.reg == CFI_SP && 2633 op->dest.reg == CFI_BP && cfi->drap) { 2634 2635 /* drap: mov %rsp, %rbp */ 2636 regs[CFI_BP].base = CFI_BP; 2637 regs[CFI_BP].offset = -cfi->stack_size; 2638 cfi->bp_scratch = false; 2639 } 2640 2641 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2642 2643 /* 2644 * mov %rsp, %reg 2645 * 2646 * This is needed for the rare case where GCC 2647 * does: 2648 * 2649 * mov %rsp, %rax 2650 * ... 2651 * mov %rax, %rsp 2652 */ 2653 cfi->vals[op->dest.reg].base = CFI_CFA; 2654 cfi->vals[op->dest.reg].offset = -cfi->stack_size; 2655 } 2656 2657 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP && 2658 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) { 2659 2660 /* 2661 * mov %rbp, %rsp 2662 * 2663 * Restore the original stack pointer (Clang). 2664 */ 2665 cfi->stack_size = -cfi->regs[CFI_BP].offset; 2666 } 2667 2668 else if (op->dest.reg == cfa->base) { 2669 2670 /* mov %reg, %rsp */ 2671 if (cfa->base == CFI_SP && 2672 cfi->vals[op->src.reg].base == CFI_CFA) { 2673 2674 /* 2675 * This is needed for the rare case 2676 * where GCC does something dumb like: 2677 * 2678 * lea 0x8(%rsp), %rcx 2679 * ... 2680 * mov %rcx, %rsp 2681 */ 2682 cfa->offset = -cfi->vals[op->src.reg].offset; 2683 cfi->stack_size = cfa->offset; 2684 2685 } else if (cfa->base == CFI_SP && 2686 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2687 cfi->vals[op->src.reg].offset == cfa->offset) { 2688 2689 /* 2690 * Stack swizzle: 2691 * 2692 * 1: mov %rsp, (%[tos]) 2693 * 2: mov %[tos], %rsp 2694 * ... 2695 * 3: pop %rsp 2696 * 2697 * Where: 2698 * 2699 * 1 - places a pointer to the previous 2700 * stack at the Top-of-Stack of the 2701 * new stack. 2702 * 2703 * 2 - switches to the new stack. 2704 * 2705 * 3 - pops the Top-of-Stack to restore 2706 * the original stack. 2707 * 2708 * Note: we set base to SP_INDIRECT 2709 * here and preserve offset. Therefore 2710 * when the unwinder reaches ToS it 2711 * will dereference SP and then add the 2712 * offset to find the next frame, IOW: 2713 * (%rsp) + offset. 2714 */ 2715 cfa->base = CFI_SP_INDIRECT; 2716 2717 } else { 2718 cfa->base = CFI_UNDEFINED; 2719 cfa->offset = 0; 2720 } 2721 } 2722 2723 else if (op->dest.reg == CFI_SP && 2724 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2725 cfi->vals[op->src.reg].offset == cfa->offset) { 2726 2727 /* 2728 * The same stack swizzle case 2) as above. But 2729 * because we can't change cfa->base, case 3) 2730 * will become a regular POP. Pretend we're a 2731 * PUSH so things don't go unbalanced. 2732 */ 2733 cfi->stack_size += 8; 2734 } 2735 2736 2737 break; 2738 2739 case OP_SRC_ADD: 2740 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) { 2741 2742 /* add imm, %rsp */ 2743 cfi->stack_size -= op->src.offset; 2744 if (cfa->base == CFI_SP) 2745 cfa->offset -= op->src.offset; 2746 break; 2747 } 2748 2749 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { 2750 2751 /* lea disp(%rbp), %rsp */ 2752 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); 2753 break; 2754 } 2755 2756 if (!cfi->drap && op->src.reg == CFI_SP && 2757 op->dest.reg == CFI_BP && cfa->base == CFI_SP && 2758 check_reg_frame_pos(®s[CFI_BP], -cfa->offset + op->src.offset)) { 2759 2760 /* lea disp(%rsp), %rbp */ 2761 cfa->base = CFI_BP; 2762 cfa->offset -= op->src.offset; 2763 cfi->bp_scratch = false; 2764 break; 2765 } 2766 2767 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2768 2769 /* drap: lea disp(%rsp), %drap */ 2770 cfi->drap_reg = op->dest.reg; 2771 2772 /* 2773 * lea disp(%rsp), %reg 2774 * 2775 * This is needed for the rare case where GCC 2776 * does something dumb like: 2777 * 2778 * lea 0x8(%rsp), %rcx 2779 * ... 2780 * mov %rcx, %rsp 2781 */ 2782 cfi->vals[op->dest.reg].base = CFI_CFA; 2783 cfi->vals[op->dest.reg].offset = \ 2784 -cfi->stack_size + op->src.offset; 2785 2786 break; 2787 } 2788 2789 if (cfi->drap && op->dest.reg == CFI_SP && 2790 op->src.reg == cfi->drap_reg) { 2791 2792 /* drap: lea disp(%drap), %rsp */ 2793 cfa->base = CFI_SP; 2794 cfa->offset = cfi->stack_size = -op->src.offset; 2795 cfi->drap_reg = CFI_UNDEFINED; 2796 cfi->drap = false; 2797 break; 2798 } 2799 2800 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { 2801 WARN_FUNC("unsupported stack register modification", 2802 insn->sec, insn->offset); 2803 return -1; 2804 } 2805 2806 break; 2807 2808 case OP_SRC_AND: 2809 if (op->dest.reg != CFI_SP || 2810 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || 2811 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { 2812 WARN_FUNC("unsupported stack pointer realignment", 2813 insn->sec, insn->offset); 2814 return -1; 2815 } 2816 2817 if (cfi->drap_reg != CFI_UNDEFINED) { 2818 /* drap: and imm, %rsp */ 2819 cfa->base = cfi->drap_reg; 2820 cfa->offset = cfi->stack_size = 0; 2821 cfi->drap = true; 2822 } 2823 2824 /* 2825 * Older versions of GCC (4.8ish) realign the stack 2826 * without DRAP, with a frame pointer. 2827 */ 2828 2829 break; 2830 2831 case OP_SRC_POP: 2832 case OP_SRC_POPF: 2833 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) { 2834 2835 /* pop %rsp; # restore from a stack swizzle */ 2836 cfa->base = CFI_SP; 2837 break; 2838 } 2839 2840 if (!cfi->drap && op->dest.reg == cfa->base) { 2841 2842 /* pop %rbp */ 2843 cfa->base = CFI_SP; 2844 } 2845 2846 if (cfi->drap && cfa->base == CFI_BP_INDIRECT && 2847 op->dest.reg == cfi->drap_reg && 2848 cfi->drap_offset == -cfi->stack_size) { 2849 2850 /* drap: pop %drap */ 2851 cfa->base = cfi->drap_reg; 2852 cfa->offset = 0; 2853 cfi->drap_offset = -1; 2854 2855 } else if (cfi->stack_size == -regs[op->dest.reg].offset) { 2856 2857 /* pop %reg */ 2858 restore_reg(cfi, op->dest.reg); 2859 } 2860 2861 cfi->stack_size -= 8; 2862 if (cfa->base == CFI_SP) 2863 cfa->offset -= 8; 2864 2865 break; 2866 2867 case OP_SRC_REG_INDIRECT: 2868 if (!cfi->drap && op->dest.reg == cfa->base && 2869 op->dest.reg == CFI_BP) { 2870 2871 /* mov disp(%rsp), %rbp */ 2872 cfa->base = CFI_SP; 2873 cfa->offset = cfi->stack_size; 2874 } 2875 2876 if (cfi->drap && op->src.reg == CFI_BP && 2877 op->src.offset == cfi->drap_offset) { 2878 2879 /* drap: mov disp(%rbp), %drap */ 2880 cfa->base = cfi->drap_reg; 2881 cfa->offset = 0; 2882 cfi->drap_offset = -1; 2883 } 2884 2885 if (cfi->drap && op->src.reg == CFI_BP && 2886 op->src.offset == regs[op->dest.reg].offset) { 2887 2888 /* drap: mov disp(%rbp), %reg */ 2889 restore_reg(cfi, op->dest.reg); 2890 2891 } else if (op->src.reg == cfa->base && 2892 op->src.offset == regs[op->dest.reg].offset + cfa->offset) { 2893 2894 /* mov disp(%rbp), %reg */ 2895 /* mov disp(%rsp), %reg */ 2896 restore_reg(cfi, op->dest.reg); 2897 2898 } else if (op->src.reg == CFI_SP && 2899 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) { 2900 2901 /* mov disp(%rsp), %reg */ 2902 restore_reg(cfi, op->dest.reg); 2903 } 2904 2905 break; 2906 2907 default: 2908 WARN_FUNC("unknown stack-related instruction", 2909 insn->sec, insn->offset); 2910 return -1; 2911 } 2912 2913 break; 2914 2915 case OP_DEST_PUSH: 2916 case OP_DEST_PUSHF: 2917 cfi->stack_size += 8; 2918 if (cfa->base == CFI_SP) 2919 cfa->offset += 8; 2920 2921 if (op->src.type != OP_SRC_REG) 2922 break; 2923 2924 if (cfi->drap) { 2925 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 2926 2927 /* drap: push %drap */ 2928 cfa->base = CFI_BP_INDIRECT; 2929 cfa->offset = -cfi->stack_size; 2930 2931 /* save drap so we know when to restore it */ 2932 cfi->drap_offset = -cfi->stack_size; 2933 2934 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) { 2935 2936 /* drap: push %rbp */ 2937 cfi->stack_size = 0; 2938 2939 } else { 2940 2941 /* drap: push %reg */ 2942 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); 2943 } 2944 2945 } else { 2946 2947 /* push %reg */ 2948 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size); 2949 } 2950 2951 /* detect when asm code uses rbp as a scratch register */ 2952 if (opts.stackval && insn->func && op->src.reg == CFI_BP && 2953 cfa->base != CFI_BP) 2954 cfi->bp_scratch = true; 2955 break; 2956 2957 case OP_DEST_REG_INDIRECT: 2958 2959 if (cfi->drap) { 2960 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 2961 2962 /* drap: mov %drap, disp(%rbp) */ 2963 cfa->base = CFI_BP_INDIRECT; 2964 cfa->offset = op->dest.offset; 2965 2966 /* save drap offset so we know when to restore it */ 2967 cfi->drap_offset = op->dest.offset; 2968 } else { 2969 2970 /* drap: mov reg, disp(%rbp) */ 2971 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); 2972 } 2973 2974 } else if (op->dest.reg == cfa->base) { 2975 2976 /* mov reg, disp(%rbp) */ 2977 /* mov reg, disp(%rsp) */ 2978 save_reg(cfi, op->src.reg, CFI_CFA, 2979 op->dest.offset - cfi->cfa.offset); 2980 2981 } else if (op->dest.reg == CFI_SP) { 2982 2983 /* mov reg, disp(%rsp) */ 2984 save_reg(cfi, op->src.reg, CFI_CFA, 2985 op->dest.offset - cfi->stack_size); 2986 2987 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) { 2988 2989 /* mov %rsp, (%reg); # setup a stack swizzle. */ 2990 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT; 2991 cfi->vals[op->dest.reg].offset = cfa->offset; 2992 } 2993 2994 break; 2995 2996 case OP_DEST_MEM: 2997 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { 2998 WARN_FUNC("unknown stack-related memory operation", 2999 insn->sec, insn->offset); 3000 return -1; 3001 } 3002 3003 /* pop mem */ 3004 cfi->stack_size -= 8; 3005 if (cfa->base == CFI_SP) 3006 cfa->offset -= 8; 3007 3008 break; 3009 3010 default: 3011 WARN_FUNC("unknown stack-related instruction", 3012 insn->sec, insn->offset); 3013 return -1; 3014 } 3015 3016 return 0; 3017 } 3018 3019 /* 3020 * The stack layouts of alternatives instructions can sometimes diverge when 3021 * they have stack modifications. That's fine as long as the potential stack 3022 * layouts don't conflict at any given potential instruction boundary. 3023 * 3024 * Flatten the CFIs of the different alternative code streams (both original 3025 * and replacement) into a single shared CFI array which can be used to detect 3026 * conflicts and nicely feed a linear array of ORC entries to the unwinder. 3027 */ 3028 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn) 3029 { 3030 struct cfi_state **alt_cfi; 3031 int group_off; 3032 3033 if (!insn->alt_group) 3034 return 0; 3035 3036 if (!insn->cfi) { 3037 WARN("CFI missing"); 3038 return -1; 3039 } 3040 3041 alt_cfi = insn->alt_group->cfi; 3042 group_off = insn->offset - insn->alt_group->first_insn->offset; 3043 3044 if (!alt_cfi[group_off]) { 3045 alt_cfi[group_off] = insn->cfi; 3046 } else { 3047 if (cficmp(alt_cfi[group_off], insn->cfi)) { 3048 WARN_FUNC("stack layout conflict in alternatives", 3049 insn->sec, insn->offset); 3050 return -1; 3051 } 3052 } 3053 3054 return 0; 3055 } 3056 3057 static int handle_insn_ops(struct instruction *insn, 3058 struct instruction *next_insn, 3059 struct insn_state *state) 3060 { 3061 struct stack_op *op; 3062 3063 list_for_each_entry(op, &insn->stack_ops, list) { 3064 3065 if (update_cfi_state(insn, next_insn, &state->cfi, op)) 3066 return 1; 3067 3068 if (!insn->alt_group) 3069 continue; 3070 3071 if (op->dest.type == OP_DEST_PUSHF) { 3072 if (!state->uaccess_stack) { 3073 state->uaccess_stack = 1; 3074 } else if (state->uaccess_stack >> 31) { 3075 WARN_FUNC("PUSHF stack exhausted", 3076 insn->sec, insn->offset); 3077 return 1; 3078 } 3079 state->uaccess_stack <<= 1; 3080 state->uaccess_stack |= state->uaccess; 3081 } 3082 3083 if (op->src.type == OP_SRC_POPF) { 3084 if (state->uaccess_stack) { 3085 state->uaccess = state->uaccess_stack & 1; 3086 state->uaccess_stack >>= 1; 3087 if (state->uaccess_stack == 1) 3088 state->uaccess_stack = 0; 3089 } 3090 } 3091 } 3092 3093 return 0; 3094 } 3095 3096 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) 3097 { 3098 struct cfi_state *cfi1 = insn->cfi; 3099 int i; 3100 3101 if (!cfi1) { 3102 WARN("CFI missing"); 3103 return false; 3104 } 3105 3106 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { 3107 3108 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d", 3109 insn->sec, insn->offset, 3110 cfi1->cfa.base, cfi1->cfa.offset, 3111 cfi2->cfa.base, cfi2->cfa.offset); 3112 3113 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) { 3114 for (i = 0; i < CFI_NUM_REGS; i++) { 3115 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], 3116 sizeof(struct cfi_reg))) 3117 continue; 3118 3119 WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", 3120 insn->sec, insn->offset, 3121 i, cfi1->regs[i].base, cfi1->regs[i].offset, 3122 i, cfi2->regs[i].base, cfi2->regs[i].offset); 3123 break; 3124 } 3125 3126 } else if (cfi1->type != cfi2->type) { 3127 3128 WARN_FUNC("stack state mismatch: type1=%d type2=%d", 3129 insn->sec, insn->offset, cfi1->type, cfi2->type); 3130 3131 } else if (cfi1->drap != cfi2->drap || 3132 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || 3133 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { 3134 3135 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", 3136 insn->sec, insn->offset, 3137 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, 3138 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); 3139 3140 } else 3141 return true; 3142 3143 return false; 3144 } 3145 3146 static inline bool func_uaccess_safe(struct symbol *func) 3147 { 3148 if (func) 3149 return func->uaccess_safe; 3150 3151 return false; 3152 } 3153 3154 static inline const char *call_dest_name(struct instruction *insn) 3155 { 3156 static char pvname[19]; 3157 struct reloc *rel; 3158 int idx; 3159 3160 if (insn->call_dest) 3161 return insn->call_dest->name; 3162 3163 rel = insn_reloc(NULL, insn); 3164 if (rel && !strcmp(rel->sym->name, "pv_ops")) { 3165 idx = (rel->addend / sizeof(void *)); 3166 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx); 3167 return pvname; 3168 } 3169 3170 return "{dynamic}"; 3171 } 3172 3173 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn) 3174 { 3175 struct symbol *target; 3176 struct reloc *rel; 3177 int idx; 3178 3179 rel = insn_reloc(file, insn); 3180 if (!rel || strcmp(rel->sym->name, "pv_ops")) 3181 return false; 3182 3183 idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *)); 3184 3185 if (file->pv_ops[idx].clean) 3186 return true; 3187 3188 file->pv_ops[idx].clean = true; 3189 3190 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) { 3191 if (!target->sec->noinstr) { 3192 WARN("pv_ops[%d]: %s", idx, target->name); 3193 file->pv_ops[idx].clean = false; 3194 } 3195 } 3196 3197 return file->pv_ops[idx].clean; 3198 } 3199 3200 static inline bool noinstr_call_dest(struct objtool_file *file, 3201 struct instruction *insn, 3202 struct symbol *func) 3203 { 3204 /* 3205 * We can't deal with indirect function calls at present; 3206 * assume they're instrumented. 3207 */ 3208 if (!func) { 3209 if (file->pv_ops) 3210 return pv_call_dest(file, insn); 3211 3212 return false; 3213 } 3214 3215 /* 3216 * If the symbol is from a noinstr section; we good. 3217 */ 3218 if (func->sec->noinstr) 3219 return true; 3220 3221 /* 3222 * The __ubsan_handle_*() calls are like WARN(), they only happen when 3223 * something 'BAD' happened. At the risk of taking the machine down, 3224 * let them proceed to get the message out. 3225 */ 3226 if (!strncmp(func->name, "__ubsan_handle_", 15)) 3227 return true; 3228 3229 return false; 3230 } 3231 3232 static int validate_call(struct objtool_file *file, 3233 struct instruction *insn, 3234 struct insn_state *state) 3235 { 3236 if (state->noinstr && state->instr <= 0 && 3237 !noinstr_call_dest(file, insn, insn->call_dest)) { 3238 WARN_FUNC("call to %s() leaves .noinstr.text section", 3239 insn->sec, insn->offset, call_dest_name(insn)); 3240 return 1; 3241 } 3242 3243 if (state->uaccess && !func_uaccess_safe(insn->call_dest)) { 3244 WARN_FUNC("call to %s() with UACCESS enabled", 3245 insn->sec, insn->offset, call_dest_name(insn)); 3246 return 1; 3247 } 3248 3249 if (state->df) { 3250 WARN_FUNC("call to %s() with DF set", 3251 insn->sec, insn->offset, call_dest_name(insn)); 3252 return 1; 3253 } 3254 3255 return 0; 3256 } 3257 3258 static int validate_sibling_call(struct objtool_file *file, 3259 struct instruction *insn, 3260 struct insn_state *state) 3261 { 3262 if (has_modified_stack_frame(insn, state)) { 3263 WARN_FUNC("sibling call from callable instruction with modified stack frame", 3264 insn->sec, insn->offset); 3265 return 1; 3266 } 3267 3268 return validate_call(file, insn, state); 3269 } 3270 3271 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 3272 { 3273 if (state->noinstr && state->instr > 0) { 3274 WARN_FUNC("return with instrumentation enabled", 3275 insn->sec, insn->offset); 3276 return 1; 3277 } 3278 3279 if (state->uaccess && !func_uaccess_safe(func)) { 3280 WARN_FUNC("return with UACCESS enabled", 3281 insn->sec, insn->offset); 3282 return 1; 3283 } 3284 3285 if (!state->uaccess && func_uaccess_safe(func)) { 3286 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function", 3287 insn->sec, insn->offset); 3288 return 1; 3289 } 3290 3291 if (state->df) { 3292 WARN_FUNC("return with DF set", 3293 insn->sec, insn->offset); 3294 return 1; 3295 } 3296 3297 if (func && has_modified_stack_frame(insn, state)) { 3298 WARN_FUNC("return with modified stack frame", 3299 insn->sec, insn->offset); 3300 return 1; 3301 } 3302 3303 if (state->cfi.bp_scratch) { 3304 WARN_FUNC("BP used as a scratch register", 3305 insn->sec, insn->offset); 3306 return 1; 3307 } 3308 3309 return 0; 3310 } 3311 3312 static struct instruction *next_insn_to_validate(struct objtool_file *file, 3313 struct instruction *insn) 3314 { 3315 struct alt_group *alt_group = insn->alt_group; 3316 3317 /* 3318 * Simulate the fact that alternatives are patched in-place. When the 3319 * end of a replacement alt_group is reached, redirect objtool flow to 3320 * the end of the original alt_group. 3321 */ 3322 if (alt_group && insn == alt_group->last_insn && alt_group->orig_group) 3323 return next_insn_same_sec(file, alt_group->orig_group->last_insn); 3324 3325 return next_insn_same_sec(file, insn); 3326 } 3327 3328 /* 3329 * Follow the branch starting at the given instruction, and recursively follow 3330 * any other branches (jumps). Meanwhile, track the frame pointer state at 3331 * each instruction and validate all the rules described in 3332 * tools/objtool/Documentation/objtool.txt. 3333 */ 3334 static int validate_branch(struct objtool_file *file, struct symbol *func, 3335 struct instruction *insn, struct insn_state state) 3336 { 3337 struct alternative *alt; 3338 struct instruction *next_insn, *prev_insn = NULL; 3339 struct section *sec; 3340 u8 visited; 3341 int ret; 3342 3343 sec = insn->sec; 3344 3345 while (1) { 3346 next_insn = next_insn_to_validate(file, insn); 3347 3348 if (func && insn->func && func != insn->func->pfunc) { 3349 /* Ignore KCFI type preambles, which always fall through */ 3350 if (!strncmp(func->name, "__cfi_", 6)) 3351 return 0; 3352 3353 WARN("%s() falls through to next function %s()", 3354 func->name, insn->func->name); 3355 return 1; 3356 } 3357 3358 if (func && insn->ignore) { 3359 WARN_FUNC("BUG: why am I validating an ignored function?", 3360 sec, insn->offset); 3361 return 1; 3362 } 3363 3364 visited = VISITED_BRANCH << state.uaccess; 3365 if (insn->visited & VISITED_BRANCH_MASK) { 3366 if (!insn->hint && !insn_cfi_match(insn, &state.cfi)) 3367 return 1; 3368 3369 if (insn->visited & visited) 3370 return 0; 3371 } else { 3372 nr_insns_visited++; 3373 } 3374 3375 if (state.noinstr) 3376 state.instr += insn->instr; 3377 3378 if (insn->hint) { 3379 if (insn->restore) { 3380 struct instruction *save_insn, *i; 3381 3382 i = insn; 3383 save_insn = NULL; 3384 3385 sym_for_each_insn_continue_reverse(file, func, i) { 3386 if (i->save) { 3387 save_insn = i; 3388 break; 3389 } 3390 } 3391 3392 if (!save_insn) { 3393 WARN_FUNC("no corresponding CFI save for CFI restore", 3394 sec, insn->offset); 3395 return 1; 3396 } 3397 3398 if (!save_insn->visited) { 3399 WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo", 3400 sec, insn->offset); 3401 return 1; 3402 } 3403 3404 insn->cfi = save_insn->cfi; 3405 nr_cfi_reused++; 3406 } 3407 3408 state.cfi = *insn->cfi; 3409 } else { 3410 /* XXX track if we actually changed state.cfi */ 3411 3412 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) { 3413 insn->cfi = prev_insn->cfi; 3414 nr_cfi_reused++; 3415 } else { 3416 insn->cfi = cfi_hash_find_or_add(&state.cfi); 3417 } 3418 } 3419 3420 insn->visited |= visited; 3421 3422 if (propagate_alt_cfi(file, insn)) 3423 return 1; 3424 3425 if (!insn->ignore_alts && !list_empty(&insn->alts)) { 3426 bool skip_orig = false; 3427 3428 list_for_each_entry(alt, &insn->alts, list) { 3429 if (alt->skip_orig) 3430 skip_orig = true; 3431 3432 ret = validate_branch(file, func, alt->insn, state); 3433 if (ret) { 3434 if (opts.backtrace) 3435 BT_FUNC("(alt)", insn); 3436 return ret; 3437 } 3438 } 3439 3440 if (skip_orig) 3441 return 0; 3442 } 3443 3444 if (handle_insn_ops(insn, next_insn, &state)) 3445 return 1; 3446 3447 switch (insn->type) { 3448 3449 case INSN_RETURN: 3450 return validate_return(func, insn, &state); 3451 3452 case INSN_CALL: 3453 case INSN_CALL_DYNAMIC: 3454 ret = validate_call(file, insn, &state); 3455 if (ret) 3456 return ret; 3457 3458 if (opts.stackval && func && !is_fentry_call(insn) && 3459 !has_valid_stack_frame(&state)) { 3460 WARN_FUNC("call without frame pointer save/setup", 3461 sec, insn->offset); 3462 return 1; 3463 } 3464 3465 if (insn->dead_end) 3466 return 0; 3467 3468 break; 3469 3470 case INSN_JUMP_CONDITIONAL: 3471 case INSN_JUMP_UNCONDITIONAL: 3472 if (is_sibling_call(insn)) { 3473 ret = validate_sibling_call(file, insn, &state); 3474 if (ret) 3475 return ret; 3476 3477 } else if (insn->jump_dest) { 3478 ret = validate_branch(file, func, 3479 insn->jump_dest, state); 3480 if (ret) { 3481 if (opts.backtrace) 3482 BT_FUNC("(branch)", insn); 3483 return ret; 3484 } 3485 } 3486 3487 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3488 return 0; 3489 3490 break; 3491 3492 case INSN_JUMP_DYNAMIC: 3493 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3494 if (is_sibling_call(insn)) { 3495 ret = validate_sibling_call(file, insn, &state); 3496 if (ret) 3497 return ret; 3498 } 3499 3500 if (insn->type == INSN_JUMP_DYNAMIC) 3501 return 0; 3502 3503 break; 3504 3505 case INSN_CONTEXT_SWITCH: 3506 if (func && (!next_insn || !next_insn->hint)) { 3507 WARN_FUNC("unsupported instruction in callable function", 3508 sec, insn->offset); 3509 return 1; 3510 } 3511 return 0; 3512 3513 case INSN_STAC: 3514 if (state.uaccess) { 3515 WARN_FUNC("recursive UACCESS enable", sec, insn->offset); 3516 return 1; 3517 } 3518 3519 state.uaccess = true; 3520 break; 3521 3522 case INSN_CLAC: 3523 if (!state.uaccess && func) { 3524 WARN_FUNC("redundant UACCESS disable", sec, insn->offset); 3525 return 1; 3526 } 3527 3528 if (func_uaccess_safe(func) && !state.uaccess_stack) { 3529 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset); 3530 return 1; 3531 } 3532 3533 state.uaccess = false; 3534 break; 3535 3536 case INSN_STD: 3537 if (state.df) { 3538 WARN_FUNC("recursive STD", sec, insn->offset); 3539 return 1; 3540 } 3541 3542 state.df = true; 3543 break; 3544 3545 case INSN_CLD: 3546 if (!state.df && func) { 3547 WARN_FUNC("redundant CLD", sec, insn->offset); 3548 return 1; 3549 } 3550 3551 state.df = false; 3552 break; 3553 3554 default: 3555 break; 3556 } 3557 3558 if (insn->dead_end) 3559 return 0; 3560 3561 if (!next_insn) { 3562 if (state.cfi.cfa.base == CFI_UNDEFINED) 3563 return 0; 3564 WARN("%s: unexpected end of section", sec->name); 3565 return 1; 3566 } 3567 3568 prev_insn = insn; 3569 insn = next_insn; 3570 } 3571 3572 return 0; 3573 } 3574 3575 static int validate_unwind_hints(struct objtool_file *file, struct section *sec) 3576 { 3577 struct instruction *insn; 3578 struct insn_state state; 3579 int ret, warnings = 0; 3580 3581 if (!file->hints) 3582 return 0; 3583 3584 init_insn_state(file, &state, sec); 3585 3586 if (sec) { 3587 insn = find_insn(file, sec, 0); 3588 if (!insn) 3589 return 0; 3590 } else { 3591 insn = list_first_entry(&file->insn_list, typeof(*insn), list); 3592 } 3593 3594 while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) { 3595 if (insn->hint && !insn->visited && !insn->ignore) { 3596 ret = validate_branch(file, insn->func, insn, state); 3597 if (ret && opts.backtrace) 3598 BT_FUNC("<=== (hint)", insn); 3599 warnings += ret; 3600 } 3601 3602 insn = list_next_entry(insn, list); 3603 } 3604 3605 return warnings; 3606 } 3607 3608 /* 3609 * Validate rethunk entry constraint: must untrain RET before the first RET. 3610 * 3611 * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes 3612 * before an actual RET instruction. 3613 */ 3614 static int validate_entry(struct objtool_file *file, struct instruction *insn) 3615 { 3616 struct instruction *next, *dest; 3617 int ret, warnings = 0; 3618 3619 for (;;) { 3620 next = next_insn_to_validate(file, insn); 3621 3622 if (insn->visited & VISITED_ENTRY) 3623 return 0; 3624 3625 insn->visited |= VISITED_ENTRY; 3626 3627 if (!insn->ignore_alts && !list_empty(&insn->alts)) { 3628 struct alternative *alt; 3629 bool skip_orig = false; 3630 3631 list_for_each_entry(alt, &insn->alts, list) { 3632 if (alt->skip_orig) 3633 skip_orig = true; 3634 3635 ret = validate_entry(file, alt->insn); 3636 if (ret) { 3637 if (opts.backtrace) 3638 BT_FUNC("(alt)", insn); 3639 return ret; 3640 } 3641 } 3642 3643 if (skip_orig) 3644 return 0; 3645 } 3646 3647 switch (insn->type) { 3648 3649 case INSN_CALL_DYNAMIC: 3650 case INSN_JUMP_DYNAMIC: 3651 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3652 WARN_FUNC("early indirect call", insn->sec, insn->offset); 3653 return 1; 3654 3655 case INSN_JUMP_UNCONDITIONAL: 3656 case INSN_JUMP_CONDITIONAL: 3657 if (!is_sibling_call(insn)) { 3658 if (!insn->jump_dest) { 3659 WARN_FUNC("unresolved jump target after linking?!?", 3660 insn->sec, insn->offset); 3661 return -1; 3662 } 3663 ret = validate_entry(file, insn->jump_dest); 3664 if (ret) { 3665 if (opts.backtrace) { 3666 BT_FUNC("(branch%s)", insn, 3667 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : ""); 3668 } 3669 return ret; 3670 } 3671 3672 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3673 return 0; 3674 3675 break; 3676 } 3677 3678 /* fallthrough */ 3679 case INSN_CALL: 3680 dest = find_insn(file, insn->call_dest->sec, 3681 insn->call_dest->offset); 3682 if (!dest) { 3683 WARN("Unresolved function after linking!?: %s", 3684 insn->call_dest->name); 3685 return -1; 3686 } 3687 3688 ret = validate_entry(file, dest); 3689 if (ret) { 3690 if (opts.backtrace) 3691 BT_FUNC("(call)", insn); 3692 return ret; 3693 } 3694 /* 3695 * If a call returns without error, it must have seen UNTRAIN_RET. 3696 * Therefore any non-error return is a success. 3697 */ 3698 return 0; 3699 3700 case INSN_RETURN: 3701 WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset); 3702 return 1; 3703 3704 case INSN_NOP: 3705 if (insn->retpoline_safe) 3706 return 0; 3707 break; 3708 3709 default: 3710 break; 3711 } 3712 3713 if (!next) { 3714 WARN_FUNC("teh end!", insn->sec, insn->offset); 3715 return -1; 3716 } 3717 insn = next; 3718 } 3719 3720 return warnings; 3721 } 3722 3723 /* 3724 * Validate that all branches starting at 'insn->entry' encounter UNRET_END 3725 * before RET. 3726 */ 3727 static int validate_unret(struct objtool_file *file) 3728 { 3729 struct instruction *insn; 3730 int ret, warnings = 0; 3731 3732 for_each_insn(file, insn) { 3733 if (!insn->entry) 3734 continue; 3735 3736 ret = validate_entry(file, insn); 3737 if (ret < 0) { 3738 WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset); 3739 return ret; 3740 } 3741 warnings += ret; 3742 } 3743 3744 return warnings; 3745 } 3746 3747 static int validate_retpoline(struct objtool_file *file) 3748 { 3749 struct instruction *insn; 3750 int warnings = 0; 3751 3752 for_each_insn(file, insn) { 3753 if (insn->type != INSN_JUMP_DYNAMIC && 3754 insn->type != INSN_CALL_DYNAMIC && 3755 insn->type != INSN_RETURN) 3756 continue; 3757 3758 if (insn->retpoline_safe) 3759 continue; 3760 3761 /* 3762 * .init.text code is ran before userspace and thus doesn't 3763 * strictly need retpolines, except for modules which are 3764 * loaded late, they very much do need retpoline in their 3765 * .init.text 3766 */ 3767 if (!strcmp(insn->sec->name, ".init.text") && !opts.module) 3768 continue; 3769 3770 if (insn->type == INSN_RETURN) { 3771 if (opts.rethunk) { 3772 WARN_FUNC("'naked' return found in RETHUNK build", 3773 insn->sec, insn->offset); 3774 } else 3775 continue; 3776 } else { 3777 WARN_FUNC("indirect %s found in RETPOLINE build", 3778 insn->sec, insn->offset, 3779 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); 3780 } 3781 3782 warnings++; 3783 } 3784 3785 return warnings; 3786 } 3787 3788 static bool is_kasan_insn(struct instruction *insn) 3789 { 3790 return (insn->type == INSN_CALL && 3791 !strcmp(insn->call_dest->name, "__asan_handle_no_return")); 3792 } 3793 3794 static bool is_ubsan_insn(struct instruction *insn) 3795 { 3796 return (insn->type == INSN_CALL && 3797 !strcmp(insn->call_dest->name, 3798 "__ubsan_handle_builtin_unreachable")); 3799 } 3800 3801 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) 3802 { 3803 int i; 3804 struct instruction *prev_insn; 3805 3806 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP) 3807 return true; 3808 3809 /* 3810 * Ignore alternative replacement instructions. This can happen 3811 * when a whitelisted function uses one of the ALTERNATIVE macros. 3812 */ 3813 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 3814 !strcmp(insn->sec->name, ".altinstr_aux")) 3815 return true; 3816 3817 /* 3818 * Whole archive runs might encounter dead code from weak symbols. 3819 * This is where the linker will have dropped the weak symbol in 3820 * favour of a regular symbol, but leaves the code in place. 3821 * 3822 * In this case we'll find a piece of code (whole function) that is not 3823 * covered by a !section symbol. Ignore them. 3824 */ 3825 if (opts.link && !insn->func) { 3826 int size = find_symbol_hole_containing(insn->sec, insn->offset); 3827 unsigned long end = insn->offset + size; 3828 3829 if (!size) /* not a hole */ 3830 return false; 3831 3832 if (size < 0) /* hole until the end */ 3833 return true; 3834 3835 sec_for_each_insn_continue(file, insn) { 3836 /* 3837 * If we reach a visited instruction at or before the 3838 * end of the hole, ignore the unreachable. 3839 */ 3840 if (insn->visited) 3841 return true; 3842 3843 if (insn->offset >= end) 3844 break; 3845 3846 /* 3847 * If this hole jumps to a .cold function, mark it ignore too. 3848 */ 3849 if (insn->jump_dest && insn->jump_dest->func && 3850 strstr(insn->jump_dest->func->name, ".cold")) { 3851 struct instruction *dest = insn->jump_dest; 3852 func_for_each_insn(file, dest->func, dest) 3853 dest->ignore = true; 3854 } 3855 } 3856 3857 return false; 3858 } 3859 3860 if (!insn->func) 3861 return false; 3862 3863 if (insn->func->static_call_tramp) 3864 return true; 3865 3866 /* 3867 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees 3868 * __builtin_unreachable(). The BUG() macro has an unreachable() after 3869 * the UD2, which causes GCC's undefined trap logic to emit another UD2 3870 * (or occasionally a JMP to UD2). 3871 * 3872 * It may also insert a UD2 after calling a __noreturn function. 3873 */ 3874 prev_insn = list_prev_entry(insn, list); 3875 if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) && 3876 (insn->type == INSN_BUG || 3877 (insn->type == INSN_JUMP_UNCONDITIONAL && 3878 insn->jump_dest && insn->jump_dest->type == INSN_BUG))) 3879 return true; 3880 3881 /* 3882 * Check if this (or a subsequent) instruction is related to 3883 * CONFIG_UBSAN or CONFIG_KASAN. 3884 * 3885 * End the search at 5 instructions to avoid going into the weeds. 3886 */ 3887 for (i = 0; i < 5; i++) { 3888 3889 if (is_kasan_insn(insn) || is_ubsan_insn(insn)) 3890 return true; 3891 3892 if (insn->type == INSN_JUMP_UNCONDITIONAL) { 3893 if (insn->jump_dest && 3894 insn->jump_dest->func == insn->func) { 3895 insn = insn->jump_dest; 3896 continue; 3897 } 3898 3899 break; 3900 } 3901 3902 if (insn->offset + insn->len >= insn->func->offset + insn->func->len) 3903 break; 3904 3905 insn = list_next_entry(insn, list); 3906 } 3907 3908 return false; 3909 } 3910 3911 static int validate_symbol(struct objtool_file *file, struct section *sec, 3912 struct symbol *sym, struct insn_state *state) 3913 { 3914 struct instruction *insn; 3915 int ret; 3916 3917 if (!sym->len) { 3918 WARN("%s() is missing an ELF size annotation", sym->name); 3919 return 1; 3920 } 3921 3922 if (sym->pfunc != sym || sym->alias != sym) 3923 return 0; 3924 3925 insn = find_insn(file, sec, sym->offset); 3926 if (!insn || insn->ignore || insn->visited) 3927 return 0; 3928 3929 state->uaccess = sym->uaccess_safe; 3930 3931 ret = validate_branch(file, insn->func, insn, *state); 3932 if (ret && opts.backtrace) 3933 BT_FUNC("<=== (sym)", insn); 3934 return ret; 3935 } 3936 3937 static int validate_section(struct objtool_file *file, struct section *sec) 3938 { 3939 struct insn_state state; 3940 struct symbol *func; 3941 int warnings = 0; 3942 3943 list_for_each_entry(func, &sec->symbol_list, list) { 3944 if (func->type != STT_FUNC) 3945 continue; 3946 3947 init_insn_state(file, &state, sec); 3948 set_func_state(&state.cfi); 3949 3950 warnings += validate_symbol(file, sec, func, &state); 3951 } 3952 3953 return warnings; 3954 } 3955 3956 static int validate_noinstr_sections(struct objtool_file *file) 3957 { 3958 struct section *sec; 3959 int warnings = 0; 3960 3961 sec = find_section_by_name(file->elf, ".noinstr.text"); 3962 if (sec) { 3963 warnings += validate_section(file, sec); 3964 warnings += validate_unwind_hints(file, sec); 3965 } 3966 3967 sec = find_section_by_name(file->elf, ".entry.text"); 3968 if (sec) { 3969 warnings += validate_section(file, sec); 3970 warnings += validate_unwind_hints(file, sec); 3971 } 3972 3973 return warnings; 3974 } 3975 3976 static int validate_functions(struct objtool_file *file) 3977 { 3978 struct section *sec; 3979 int warnings = 0; 3980 3981 for_each_sec(file, sec) { 3982 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 3983 continue; 3984 3985 warnings += validate_section(file, sec); 3986 } 3987 3988 return warnings; 3989 } 3990 3991 static void mark_endbr_used(struct instruction *insn) 3992 { 3993 if (!list_empty(&insn->call_node)) 3994 list_del_init(&insn->call_node); 3995 } 3996 3997 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn) 3998 { 3999 struct instruction *dest; 4000 struct reloc *reloc; 4001 unsigned long off; 4002 int warnings = 0; 4003 4004 /* 4005 * Looking for function pointer load relocations. Ignore 4006 * direct/indirect branches: 4007 */ 4008 switch (insn->type) { 4009 case INSN_CALL: 4010 case INSN_CALL_DYNAMIC: 4011 case INSN_JUMP_CONDITIONAL: 4012 case INSN_JUMP_UNCONDITIONAL: 4013 case INSN_JUMP_DYNAMIC: 4014 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4015 case INSN_RETURN: 4016 case INSN_NOP: 4017 return 0; 4018 default: 4019 break; 4020 } 4021 4022 for (reloc = insn_reloc(file, insn); 4023 reloc; 4024 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 4025 reloc->offset + 1, 4026 (insn->offset + insn->len) - (reloc->offset + 1))) { 4027 4028 /* 4029 * static_call_update() references the trampoline, which 4030 * doesn't have (or need) ENDBR. Skip warning in that case. 4031 */ 4032 if (reloc->sym->static_call_tramp) 4033 continue; 4034 4035 off = reloc->sym->offset; 4036 if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32) 4037 off += arch_dest_reloc_offset(reloc->addend); 4038 else 4039 off += reloc->addend; 4040 4041 dest = find_insn(file, reloc->sym->sec, off); 4042 if (!dest) 4043 continue; 4044 4045 if (dest->type == INSN_ENDBR) { 4046 mark_endbr_used(dest); 4047 continue; 4048 } 4049 4050 if (dest->func && dest->func == insn->func) { 4051 /* 4052 * Anything from->to self is either _THIS_IP_ or 4053 * IRET-to-self. 4054 * 4055 * There is no sane way to annotate _THIS_IP_ since the 4056 * compiler treats the relocation as a constant and is 4057 * happy to fold in offsets, skewing any annotation we 4058 * do, leading to vast amounts of false-positives. 4059 * 4060 * There's also compiler generated _THIS_IP_ through 4061 * KCOV and such which we have no hope of annotating. 4062 * 4063 * As such, blanket accept self-references without 4064 * issue. 4065 */ 4066 continue; 4067 } 4068 4069 if (dest->noendbr) 4070 continue; 4071 4072 WARN_FUNC("relocation to !ENDBR: %s", 4073 insn->sec, insn->offset, 4074 offstr(dest->sec, dest->offset)); 4075 4076 warnings++; 4077 } 4078 4079 return warnings; 4080 } 4081 4082 static int validate_ibt_data_reloc(struct objtool_file *file, 4083 struct reloc *reloc) 4084 { 4085 struct instruction *dest; 4086 4087 dest = find_insn(file, reloc->sym->sec, 4088 reloc->sym->offset + reloc->addend); 4089 if (!dest) 4090 return 0; 4091 4092 if (dest->type == INSN_ENDBR) { 4093 mark_endbr_used(dest); 4094 return 0; 4095 } 4096 4097 if (dest->noendbr) 4098 return 0; 4099 4100 WARN_FUNC("data relocation to !ENDBR: %s", 4101 reloc->sec->base, reloc->offset, 4102 offstr(dest->sec, dest->offset)); 4103 4104 return 1; 4105 } 4106 4107 /* 4108 * Validate IBT rules and remove used ENDBR instructions from the seal list. 4109 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with 4110 * NOPs) later, in create_ibt_endbr_seal_sections(). 4111 */ 4112 static int validate_ibt(struct objtool_file *file) 4113 { 4114 struct section *sec; 4115 struct reloc *reloc; 4116 struct instruction *insn; 4117 int warnings = 0; 4118 4119 for_each_insn(file, insn) 4120 warnings += validate_ibt_insn(file, insn); 4121 4122 for_each_sec(file, sec) { 4123 4124 /* Already done by validate_ibt_insn() */ 4125 if (sec->sh.sh_flags & SHF_EXECINSTR) 4126 continue; 4127 4128 if (!sec->reloc) 4129 continue; 4130 4131 /* 4132 * These sections can reference text addresses, but not with 4133 * the intent to indirect branch to them. 4134 */ 4135 if ((!strncmp(sec->name, ".discard", 8) && 4136 strcmp(sec->name, ".discard.ibt_endbr_noseal")) || 4137 !strncmp(sec->name, ".debug", 6) || 4138 !strcmp(sec->name, ".altinstructions") || 4139 !strcmp(sec->name, ".ibt_endbr_seal") || 4140 !strcmp(sec->name, ".orc_unwind_ip") || 4141 !strcmp(sec->name, ".parainstructions") || 4142 !strcmp(sec->name, ".retpoline_sites") || 4143 !strcmp(sec->name, ".smp_locks") || 4144 !strcmp(sec->name, ".static_call_sites") || 4145 !strcmp(sec->name, "_error_injection_whitelist") || 4146 !strcmp(sec->name, "_kprobe_blacklist") || 4147 !strcmp(sec->name, "__bug_table") || 4148 !strcmp(sec->name, "__ex_table") || 4149 !strcmp(sec->name, "__jump_table") || 4150 !strcmp(sec->name, "__mcount_loc") || 4151 !strcmp(sec->name, ".kcfi_traps") || 4152 strstr(sec->name, "__patchable_function_entries")) 4153 continue; 4154 4155 list_for_each_entry(reloc, &sec->reloc->reloc_list, list) 4156 warnings += validate_ibt_data_reloc(file, reloc); 4157 } 4158 4159 return warnings; 4160 } 4161 4162 static int validate_sls(struct objtool_file *file) 4163 { 4164 struct instruction *insn, *next_insn; 4165 int warnings = 0; 4166 4167 for_each_insn(file, insn) { 4168 next_insn = next_insn_same_sec(file, insn); 4169 4170 if (insn->retpoline_safe) 4171 continue; 4172 4173 switch (insn->type) { 4174 case INSN_RETURN: 4175 if (!next_insn || next_insn->type != INSN_TRAP) { 4176 WARN_FUNC("missing int3 after ret", 4177 insn->sec, insn->offset); 4178 warnings++; 4179 } 4180 4181 break; 4182 case INSN_JUMP_DYNAMIC: 4183 if (!next_insn || next_insn->type != INSN_TRAP) { 4184 WARN_FUNC("missing int3 after indirect jump", 4185 insn->sec, insn->offset); 4186 warnings++; 4187 } 4188 break; 4189 default: 4190 break; 4191 } 4192 } 4193 4194 return warnings; 4195 } 4196 4197 static int validate_reachable_instructions(struct objtool_file *file) 4198 { 4199 struct instruction *insn; 4200 4201 if (file->ignore_unreachables) 4202 return 0; 4203 4204 for_each_insn(file, insn) { 4205 if (insn->visited || ignore_unreachable_insn(file, insn)) 4206 continue; 4207 4208 WARN_FUNC("unreachable instruction", insn->sec, insn->offset); 4209 return 1; 4210 } 4211 4212 return 0; 4213 } 4214 4215 int check(struct objtool_file *file) 4216 { 4217 int ret, warnings = 0; 4218 4219 arch_initial_func_cfi_state(&initial_func_cfi); 4220 init_cfi_state(&init_cfi); 4221 init_cfi_state(&func_cfi); 4222 set_func_state(&func_cfi); 4223 4224 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) 4225 goto out; 4226 4227 cfi_hash_add(&init_cfi); 4228 cfi_hash_add(&func_cfi); 4229 4230 ret = decode_sections(file); 4231 if (ret < 0) 4232 goto out; 4233 4234 warnings += ret; 4235 4236 if (list_empty(&file->insn_list)) 4237 goto out; 4238 4239 if (opts.retpoline) { 4240 ret = validate_retpoline(file); 4241 if (ret < 0) 4242 return ret; 4243 warnings += ret; 4244 } 4245 4246 if (opts.stackval || opts.orc || opts.uaccess) { 4247 ret = validate_functions(file); 4248 if (ret < 0) 4249 goto out; 4250 warnings += ret; 4251 4252 ret = validate_unwind_hints(file, NULL); 4253 if (ret < 0) 4254 goto out; 4255 warnings += ret; 4256 4257 if (!warnings) { 4258 ret = validate_reachable_instructions(file); 4259 if (ret < 0) 4260 goto out; 4261 warnings += ret; 4262 } 4263 4264 } else if (opts.noinstr) { 4265 ret = validate_noinstr_sections(file); 4266 if (ret < 0) 4267 goto out; 4268 warnings += ret; 4269 } 4270 4271 if (opts.unret) { 4272 /* 4273 * Must be after validate_branch() and friends, it plays 4274 * further games with insn->visited. 4275 */ 4276 ret = validate_unret(file); 4277 if (ret < 0) 4278 return ret; 4279 warnings += ret; 4280 } 4281 4282 if (opts.ibt) { 4283 ret = validate_ibt(file); 4284 if (ret < 0) 4285 goto out; 4286 warnings += ret; 4287 } 4288 4289 if (opts.sls) { 4290 ret = validate_sls(file); 4291 if (ret < 0) 4292 goto out; 4293 warnings += ret; 4294 } 4295 4296 if (opts.static_call) { 4297 ret = create_static_call_sections(file); 4298 if (ret < 0) 4299 goto out; 4300 warnings += ret; 4301 } 4302 4303 if (opts.retpoline) { 4304 ret = create_retpoline_sites_sections(file); 4305 if (ret < 0) 4306 goto out; 4307 warnings += ret; 4308 } 4309 4310 if (opts.rethunk) { 4311 ret = create_return_sites_sections(file); 4312 if (ret < 0) 4313 goto out; 4314 warnings += ret; 4315 } 4316 4317 if (opts.mcount) { 4318 ret = create_mcount_loc_sections(file); 4319 if (ret < 0) 4320 goto out; 4321 warnings += ret; 4322 } 4323 4324 if (opts.ibt) { 4325 ret = create_ibt_endbr_seal_sections(file); 4326 if (ret < 0) 4327 goto out; 4328 warnings += ret; 4329 } 4330 4331 if (opts.orc && !list_empty(&file->insn_list)) { 4332 ret = orc_create(file); 4333 if (ret < 0) 4334 goto out; 4335 warnings += ret; 4336 } 4337 4338 4339 if (opts.stats) { 4340 printf("nr_insns_visited: %ld\n", nr_insns_visited); 4341 printf("nr_cfi: %ld\n", nr_cfi); 4342 printf("nr_cfi_reused: %ld\n", nr_cfi_reused); 4343 printf("nr_cfi_cache: %ld\n", nr_cfi_cache); 4344 } 4345 4346 out: 4347 /* 4348 * For now, don't fail the kernel build on fatal warnings. These 4349 * errors are still fairly common due to the growing matrix of 4350 * supported toolchains and their recent pace of change. 4351 */ 4352 return 0; 4353 } 4354