1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6 #include <string.h> 7 #include <stdlib.h> 8 #include <sys/mman.h> 9 10 #include <arch/elf.h> 11 #include <objtool/builtin.h> 12 #include <objtool/cfi.h> 13 #include <objtool/arch.h> 14 #include <objtool/check.h> 15 #include <objtool/special.h> 16 #include <objtool/warn.h> 17 #include <objtool/endianness.h> 18 19 #include <linux/objtool.h> 20 #include <linux/hashtable.h> 21 #include <linux/kernel.h> 22 #include <linux/static_call_types.h> 23 24 struct alternative { 25 struct list_head list; 26 struct instruction *insn; 27 bool skip_orig; 28 }; 29 30 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 31 32 static struct cfi_init_state initial_func_cfi; 33 static struct cfi_state init_cfi; 34 static struct cfi_state func_cfi; 35 36 struct instruction *find_insn(struct objtool_file *file, 37 struct section *sec, unsigned long offset) 38 { 39 struct instruction *insn; 40 41 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) { 42 if (insn->sec == sec && insn->offset == offset) 43 return insn; 44 } 45 46 return NULL; 47 } 48 49 static struct instruction *next_insn_same_sec(struct objtool_file *file, 50 struct instruction *insn) 51 { 52 struct instruction *next = list_next_entry(insn, list); 53 54 if (!next || &next->list == &file->insn_list || next->sec != insn->sec) 55 return NULL; 56 57 return next; 58 } 59 60 static struct instruction *next_insn_same_func(struct objtool_file *file, 61 struct instruction *insn) 62 { 63 struct instruction *next = list_next_entry(insn, list); 64 struct symbol *func = insn->func; 65 66 if (!func) 67 return NULL; 68 69 if (&next->list != &file->insn_list && next->func == func) 70 return next; 71 72 /* Check if we're already in the subfunction: */ 73 if (func == func->cfunc) 74 return NULL; 75 76 /* Move to the subfunction: */ 77 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 78 } 79 80 static struct instruction *prev_insn_same_sym(struct objtool_file *file, 81 struct instruction *insn) 82 { 83 struct instruction *prev = list_prev_entry(insn, list); 84 85 if (&prev->list != &file->insn_list && prev->func == insn->func) 86 return prev; 87 88 return NULL; 89 } 90 91 #define func_for_each_insn(file, func, insn) \ 92 for (insn = find_insn(file, func->sec, func->offset); \ 93 insn; \ 94 insn = next_insn_same_func(file, insn)) 95 96 #define sym_for_each_insn(file, sym, insn) \ 97 for (insn = find_insn(file, sym->sec, sym->offset); \ 98 insn && &insn->list != &file->insn_list && \ 99 insn->sec == sym->sec && \ 100 insn->offset < sym->offset + sym->len; \ 101 insn = list_next_entry(insn, list)) 102 103 #define sym_for_each_insn_continue_reverse(file, sym, insn) \ 104 for (insn = list_prev_entry(insn, list); \ 105 &insn->list != &file->insn_list && \ 106 insn->sec == sym->sec && insn->offset >= sym->offset; \ 107 insn = list_prev_entry(insn, list)) 108 109 #define sec_for_each_insn_from(file, insn) \ 110 for (; insn; insn = next_insn_same_sec(file, insn)) 111 112 #define sec_for_each_insn_continue(file, insn) \ 113 for (insn = next_insn_same_sec(file, insn); insn; \ 114 insn = next_insn_same_sec(file, insn)) 115 116 static bool is_jump_table_jump(struct instruction *insn) 117 { 118 struct alt_group *alt_group = insn->alt_group; 119 120 if (insn->jump_table) 121 return true; 122 123 /* Retpoline alternative for a jump table? */ 124 return alt_group && alt_group->orig_group && 125 alt_group->orig_group->first_insn->jump_table; 126 } 127 128 static bool is_sibling_call(struct instruction *insn) 129 { 130 /* 131 * Assume only ELF functions can make sibling calls. This ensures 132 * sibling call detection consistency between vmlinux.o and individual 133 * objects. 134 */ 135 if (!insn->func) 136 return false; 137 138 /* An indirect jump is either a sibling call or a jump to a table. */ 139 if (insn->type == INSN_JUMP_DYNAMIC) 140 return !is_jump_table_jump(insn); 141 142 /* add_jump_destinations() sets insn->call_dest for sibling calls. */ 143 return (is_static_jump(insn) && insn->call_dest); 144 } 145 146 /* 147 * This checks to see if the given function is a "noreturn" function. 148 * 149 * For global functions which are outside the scope of this object file, we 150 * have to keep a manual list of them. 151 * 152 * For local functions, we have to detect them manually by simply looking for 153 * the lack of a return instruction. 154 */ 155 static bool __dead_end_function(struct objtool_file *file, struct symbol *func, 156 int recursion) 157 { 158 int i; 159 struct instruction *insn; 160 bool empty = true; 161 162 /* 163 * Unfortunately these have to be hard coded because the noreturn 164 * attribute isn't provided in ELF data. 165 */ 166 static const char * const global_noreturns[] = { 167 "__stack_chk_fail", 168 "panic", 169 "do_exit", 170 "do_task_dead", 171 "kthread_exit", 172 "make_task_dead", 173 "__module_put_and_kthread_exit", 174 "kthread_complete_and_exit", 175 "__reiserfs_panic", 176 "lbug_with_loc", 177 "fortify_panic", 178 "usercopy_abort", 179 "machine_real_restart", 180 "rewind_stack_and_make_dead", 181 "kunit_try_catch_throw", 182 "xen_start_kernel", 183 "cpu_bringup_and_idle", 184 "do_group_exit", 185 "stop_this_cpu", 186 "__invalid_creds", 187 }; 188 189 if (!func) 190 return false; 191 192 if (func->bind == STB_WEAK) 193 return false; 194 195 if (func->bind == STB_GLOBAL) 196 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) 197 if (!strcmp(func->name, global_noreturns[i])) 198 return true; 199 200 if (!func->len) 201 return false; 202 203 insn = find_insn(file, func->sec, func->offset); 204 if (!insn->func) 205 return false; 206 207 func_for_each_insn(file, func, insn) { 208 empty = false; 209 210 if (insn->type == INSN_RETURN) 211 return false; 212 } 213 214 if (empty) 215 return false; 216 217 /* 218 * A function can have a sibling call instead of a return. In that 219 * case, the function's dead-end status depends on whether the target 220 * of the sibling call returns. 221 */ 222 func_for_each_insn(file, func, insn) { 223 if (is_sibling_call(insn)) { 224 struct instruction *dest = insn->jump_dest; 225 226 if (!dest) 227 /* sibling call to another file */ 228 return false; 229 230 /* local sibling call */ 231 if (recursion == 5) { 232 /* 233 * Infinite recursion: two functions have 234 * sibling calls to each other. This is a very 235 * rare case. It means they aren't dead ends. 236 */ 237 return false; 238 } 239 240 return __dead_end_function(file, dest->func, recursion+1); 241 } 242 } 243 244 return true; 245 } 246 247 static bool dead_end_function(struct objtool_file *file, struct symbol *func) 248 { 249 return __dead_end_function(file, func, 0); 250 } 251 252 static void init_cfi_state(struct cfi_state *cfi) 253 { 254 int i; 255 256 for (i = 0; i < CFI_NUM_REGS; i++) { 257 cfi->regs[i].base = CFI_UNDEFINED; 258 cfi->vals[i].base = CFI_UNDEFINED; 259 } 260 cfi->cfa.base = CFI_UNDEFINED; 261 cfi->drap_reg = CFI_UNDEFINED; 262 cfi->drap_offset = -1; 263 } 264 265 static void init_insn_state(struct insn_state *state, struct section *sec) 266 { 267 memset(state, 0, sizeof(*state)); 268 init_cfi_state(&state->cfi); 269 270 /* 271 * We need the full vmlinux for noinstr validation, otherwise we can 272 * not correctly determine insn->call_dest->sec (external symbols do 273 * not have a section). 274 */ 275 if (vmlinux && noinstr && sec) 276 state->noinstr = sec->noinstr; 277 } 278 279 static struct cfi_state *cfi_alloc(void) 280 { 281 struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1); 282 if (!cfi) { 283 WARN("calloc failed"); 284 exit(1); 285 } 286 nr_cfi++; 287 return cfi; 288 } 289 290 static int cfi_bits; 291 static struct hlist_head *cfi_hash; 292 293 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2) 294 { 295 return memcmp((void *)cfi1 + sizeof(cfi1->hash), 296 (void *)cfi2 + sizeof(cfi2->hash), 297 sizeof(struct cfi_state) - sizeof(struct hlist_node)); 298 } 299 300 static inline u32 cfi_key(struct cfi_state *cfi) 301 { 302 return jhash((void *)cfi + sizeof(cfi->hash), 303 sizeof(*cfi) - sizeof(cfi->hash), 0); 304 } 305 306 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi) 307 { 308 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 309 struct cfi_state *obj; 310 311 hlist_for_each_entry(obj, head, hash) { 312 if (!cficmp(cfi, obj)) { 313 nr_cfi_cache++; 314 return obj; 315 } 316 } 317 318 obj = cfi_alloc(); 319 *obj = *cfi; 320 hlist_add_head(&obj->hash, head); 321 322 return obj; 323 } 324 325 static void cfi_hash_add(struct cfi_state *cfi) 326 { 327 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 328 329 hlist_add_head(&cfi->hash, head); 330 } 331 332 static void *cfi_hash_alloc(unsigned long size) 333 { 334 cfi_bits = max(10, ilog2(size)); 335 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits, 336 PROT_READ|PROT_WRITE, 337 MAP_PRIVATE|MAP_ANON, -1, 0); 338 if (cfi_hash == (void *)-1L) { 339 WARN("mmap fail cfi_hash"); 340 cfi_hash = NULL; 341 } else if (stats) { 342 printf("cfi_bits: %d\n", cfi_bits); 343 } 344 345 return cfi_hash; 346 } 347 348 static unsigned long nr_insns; 349 static unsigned long nr_insns_visited; 350 351 /* 352 * Call the arch-specific instruction decoder for all the instructions and add 353 * them to the global instruction list. 354 */ 355 static int decode_instructions(struct objtool_file *file) 356 { 357 struct section *sec; 358 struct symbol *func; 359 unsigned long offset; 360 struct instruction *insn; 361 int ret; 362 363 for_each_sec(file, sec) { 364 365 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 366 continue; 367 368 if (strcmp(sec->name, ".altinstr_replacement") && 369 strcmp(sec->name, ".altinstr_aux") && 370 strncmp(sec->name, ".discard.", 9)) 371 sec->text = true; 372 373 if (!strcmp(sec->name, ".noinstr.text") || 374 !strcmp(sec->name, ".entry.text")) 375 sec->noinstr = true; 376 377 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) { 378 insn = malloc(sizeof(*insn)); 379 if (!insn) { 380 WARN("malloc failed"); 381 return -1; 382 } 383 memset(insn, 0, sizeof(*insn)); 384 INIT_LIST_HEAD(&insn->alts); 385 INIT_LIST_HEAD(&insn->stack_ops); 386 INIT_LIST_HEAD(&insn->call_node); 387 388 insn->sec = sec; 389 insn->offset = offset; 390 391 ret = arch_decode_instruction(file, sec, offset, 392 sec->sh.sh_size - offset, 393 &insn->len, &insn->type, 394 &insn->immediate, 395 &insn->stack_ops); 396 if (ret) 397 goto err; 398 399 /* 400 * By default, "ud2" is a dead end unless otherwise 401 * annotated, because GCC 7 inserts it for certain 402 * divide-by-zero cases. 403 */ 404 if (insn->type == INSN_BUG) 405 insn->dead_end = true; 406 407 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset)); 408 list_add_tail(&insn->list, &file->insn_list); 409 nr_insns++; 410 } 411 412 list_for_each_entry(func, &sec->symbol_list, list) { 413 if (func->type != STT_FUNC || func->alias != func) 414 continue; 415 416 if (!find_insn(file, sec, func->offset)) { 417 WARN("%s(): can't find starting instruction", 418 func->name); 419 return -1; 420 } 421 422 sym_for_each_insn(file, func, insn) { 423 insn->func = func; 424 if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) { 425 if (insn->offset == insn->func->offset) { 426 list_add_tail(&insn->call_node, &file->endbr_list); 427 file->nr_endbr++; 428 } else { 429 file->nr_endbr_int++; 430 } 431 } 432 } 433 } 434 } 435 436 if (stats) 437 printf("nr_insns: %lu\n", nr_insns); 438 439 return 0; 440 441 err: 442 free(insn); 443 return ret; 444 } 445 446 /* 447 * Read the pv_ops[] .data table to find the static initialized values. 448 */ 449 static int add_pv_ops(struct objtool_file *file, const char *symname) 450 { 451 struct symbol *sym, *func; 452 unsigned long off, end; 453 struct reloc *rel; 454 int idx; 455 456 sym = find_symbol_by_name(file->elf, symname); 457 if (!sym) 458 return 0; 459 460 off = sym->offset; 461 end = off + sym->len; 462 for (;;) { 463 rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 464 if (!rel) 465 break; 466 467 func = rel->sym; 468 if (func->type == STT_SECTION) 469 func = find_symbol_by_offset(rel->sym->sec, rel->addend); 470 471 idx = (rel->offset - sym->offset) / sizeof(unsigned long); 472 473 objtool_pv_add(file, idx, func); 474 475 off = rel->offset + 1; 476 if (off > end) 477 break; 478 } 479 480 return 0; 481 } 482 483 /* 484 * Allocate and initialize file->pv_ops[]. 485 */ 486 static int init_pv_ops(struct objtool_file *file) 487 { 488 static const char *pv_ops_tables[] = { 489 "pv_ops", 490 "xen_cpu_ops", 491 "xen_irq_ops", 492 "xen_mmu_ops", 493 NULL, 494 }; 495 const char *pv_ops; 496 struct symbol *sym; 497 int idx, nr; 498 499 if (!noinstr) 500 return 0; 501 502 file->pv_ops = NULL; 503 504 sym = find_symbol_by_name(file->elf, "pv_ops"); 505 if (!sym) 506 return 0; 507 508 nr = sym->len / sizeof(unsigned long); 509 file->pv_ops = calloc(sizeof(struct pv_state), nr); 510 if (!file->pv_ops) 511 return -1; 512 513 for (idx = 0; idx < nr; idx++) 514 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 515 516 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) 517 add_pv_ops(file, pv_ops); 518 519 return 0; 520 } 521 522 static struct instruction *find_last_insn(struct objtool_file *file, 523 struct section *sec) 524 { 525 struct instruction *insn = NULL; 526 unsigned int offset; 527 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0; 528 529 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--) 530 insn = find_insn(file, sec, offset); 531 532 return insn; 533 } 534 535 /* 536 * Mark "ud2" instructions and manually annotated dead ends. 537 */ 538 static int add_dead_ends(struct objtool_file *file) 539 { 540 struct section *sec; 541 struct reloc *reloc; 542 struct instruction *insn; 543 544 /* 545 * Check for manually annotated dead ends. 546 */ 547 sec = find_section_by_name(file->elf, ".rela.discard.unreachable"); 548 if (!sec) 549 goto reachable; 550 551 list_for_each_entry(reloc, &sec->reloc_list, list) { 552 if (reloc->sym->type != STT_SECTION) { 553 WARN("unexpected relocation symbol type in %s", sec->name); 554 return -1; 555 } 556 insn = find_insn(file, reloc->sym->sec, reloc->addend); 557 if (insn) 558 insn = list_prev_entry(insn, list); 559 else if (reloc->addend == reloc->sym->sec->sh.sh_size) { 560 insn = find_last_insn(file, reloc->sym->sec); 561 if (!insn) { 562 WARN("can't find unreachable insn at %s+0x%x", 563 reloc->sym->sec->name, reloc->addend); 564 return -1; 565 } 566 } else { 567 WARN("can't find unreachable insn at %s+0x%x", 568 reloc->sym->sec->name, reloc->addend); 569 return -1; 570 } 571 572 insn->dead_end = true; 573 } 574 575 reachable: 576 /* 577 * These manually annotated reachable checks are needed for GCC 4.4, 578 * where the Linux unreachable() macro isn't supported. In that case 579 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's 580 * not a dead end. 581 */ 582 sec = find_section_by_name(file->elf, ".rela.discard.reachable"); 583 if (!sec) 584 return 0; 585 586 list_for_each_entry(reloc, &sec->reloc_list, list) { 587 if (reloc->sym->type != STT_SECTION) { 588 WARN("unexpected relocation symbol type in %s", sec->name); 589 return -1; 590 } 591 insn = find_insn(file, reloc->sym->sec, reloc->addend); 592 if (insn) 593 insn = list_prev_entry(insn, list); 594 else if (reloc->addend == reloc->sym->sec->sh.sh_size) { 595 insn = find_last_insn(file, reloc->sym->sec); 596 if (!insn) { 597 WARN("can't find reachable insn at %s+0x%x", 598 reloc->sym->sec->name, reloc->addend); 599 return -1; 600 } 601 } else { 602 WARN("can't find reachable insn at %s+0x%x", 603 reloc->sym->sec->name, reloc->addend); 604 return -1; 605 } 606 607 insn->dead_end = false; 608 } 609 610 return 0; 611 } 612 613 static int create_static_call_sections(struct objtool_file *file) 614 { 615 struct section *sec; 616 struct static_call_site *site; 617 struct instruction *insn; 618 struct symbol *key_sym; 619 char *key_name, *tmp; 620 int idx; 621 622 sec = find_section_by_name(file->elf, ".static_call_sites"); 623 if (sec) { 624 INIT_LIST_HEAD(&file->static_call_list); 625 WARN("file already has .static_call_sites section, skipping"); 626 return 0; 627 } 628 629 if (list_empty(&file->static_call_list)) 630 return 0; 631 632 idx = 0; 633 list_for_each_entry(insn, &file->static_call_list, call_node) 634 idx++; 635 636 sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE, 637 sizeof(struct static_call_site), idx); 638 if (!sec) 639 return -1; 640 641 idx = 0; 642 list_for_each_entry(insn, &file->static_call_list, call_node) { 643 644 site = (struct static_call_site *)sec->data->d_buf + idx; 645 memset(site, 0, sizeof(struct static_call_site)); 646 647 /* populate reloc for 'addr' */ 648 if (elf_add_reloc_to_insn(file->elf, sec, 649 idx * sizeof(struct static_call_site), 650 R_X86_64_PC32, 651 insn->sec, insn->offset)) 652 return -1; 653 654 /* find key symbol */ 655 key_name = strdup(insn->call_dest->name); 656 if (!key_name) { 657 perror("strdup"); 658 return -1; 659 } 660 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR, 661 STATIC_CALL_TRAMP_PREFIX_LEN)) { 662 WARN("static_call: trampoline name malformed: %s", key_name); 663 return -1; 664 } 665 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN; 666 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN); 667 668 key_sym = find_symbol_by_name(file->elf, tmp); 669 if (!key_sym) { 670 if (!module) { 671 WARN("static_call: can't find static_call_key symbol: %s", tmp); 672 return -1; 673 } 674 675 /* 676 * For modules(), the key might not be exported, which 677 * means the module can make static calls but isn't 678 * allowed to change them. 679 * 680 * In that case we temporarily set the key to be the 681 * trampoline address. This is fixed up in 682 * static_call_add_module(). 683 */ 684 key_sym = insn->call_dest; 685 } 686 free(key_name); 687 688 /* populate reloc for 'key' */ 689 if (elf_add_reloc(file->elf, sec, 690 idx * sizeof(struct static_call_site) + 4, 691 R_X86_64_PC32, key_sym, 692 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL)) 693 return -1; 694 695 idx++; 696 } 697 698 return 0; 699 } 700 701 static int create_retpoline_sites_sections(struct objtool_file *file) 702 { 703 struct instruction *insn; 704 struct section *sec; 705 int idx; 706 707 sec = find_section_by_name(file->elf, ".retpoline_sites"); 708 if (sec) { 709 WARN("file already has .retpoline_sites, skipping"); 710 return 0; 711 } 712 713 idx = 0; 714 list_for_each_entry(insn, &file->retpoline_call_list, call_node) 715 idx++; 716 717 if (!idx) 718 return 0; 719 720 sec = elf_create_section(file->elf, ".retpoline_sites", 0, 721 sizeof(int), idx); 722 if (!sec) { 723 WARN("elf_create_section: .retpoline_sites"); 724 return -1; 725 } 726 727 idx = 0; 728 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 729 730 int *site = (int *)sec->data->d_buf + idx; 731 *site = 0; 732 733 if (elf_add_reloc_to_insn(file->elf, sec, 734 idx * sizeof(int), 735 R_X86_64_PC32, 736 insn->sec, insn->offset)) { 737 WARN("elf_add_reloc_to_insn: .retpoline_sites"); 738 return -1; 739 } 740 741 idx++; 742 } 743 744 return 0; 745 } 746 747 static int create_ibt_endbr_seal_sections(struct objtool_file *file) 748 { 749 struct instruction *insn; 750 struct section *sec; 751 int idx; 752 753 sec = find_section_by_name(file->elf, ".ibt_endbr_seal"); 754 if (sec) { 755 WARN("file already has .ibt_endbr_seal, skipping"); 756 return 0; 757 } 758 759 idx = 0; 760 list_for_each_entry(insn, &file->endbr_list, call_node) 761 idx++; 762 763 if (stats) { 764 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); 765 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); 766 printf("ibt: superfluous ENDBR: %d\n", idx); 767 } 768 769 if (!idx) 770 return 0; 771 772 sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0, 773 sizeof(int), idx); 774 if (!sec) { 775 WARN("elf_create_section: .ibt_endbr_seal"); 776 return -1; 777 } 778 779 idx = 0; 780 list_for_each_entry(insn, &file->endbr_list, call_node) { 781 782 int *site = (int *)sec->data->d_buf + idx; 783 *site = 0; 784 785 if (elf_add_reloc_to_insn(file->elf, sec, 786 idx * sizeof(int), 787 R_X86_64_PC32, 788 insn->sec, insn->offset)) { 789 WARN("elf_add_reloc_to_insn: .ibt_endbr_seal"); 790 return -1; 791 } 792 793 idx++; 794 } 795 796 return 0; 797 } 798 799 static int create_mcount_loc_sections(struct objtool_file *file) 800 { 801 struct section *sec; 802 unsigned long *loc; 803 struct instruction *insn; 804 int idx; 805 806 sec = find_section_by_name(file->elf, "__mcount_loc"); 807 if (sec) { 808 INIT_LIST_HEAD(&file->mcount_loc_list); 809 WARN("file already has __mcount_loc section, skipping"); 810 return 0; 811 } 812 813 if (list_empty(&file->mcount_loc_list)) 814 return 0; 815 816 idx = 0; 817 list_for_each_entry(insn, &file->mcount_loc_list, call_node) 818 idx++; 819 820 sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx); 821 if (!sec) 822 return -1; 823 824 idx = 0; 825 list_for_each_entry(insn, &file->mcount_loc_list, call_node) { 826 827 loc = (unsigned long *)sec->data->d_buf + idx; 828 memset(loc, 0, sizeof(unsigned long)); 829 830 if (elf_add_reloc_to_insn(file->elf, sec, 831 idx * sizeof(unsigned long), 832 R_X86_64_64, 833 insn->sec, insn->offset)) 834 return -1; 835 836 idx++; 837 } 838 839 return 0; 840 } 841 842 /* 843 * Warnings shouldn't be reported for ignored functions. 844 */ 845 static void add_ignores(struct objtool_file *file) 846 { 847 struct instruction *insn; 848 struct section *sec; 849 struct symbol *func; 850 struct reloc *reloc; 851 852 sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard"); 853 if (!sec) 854 return; 855 856 list_for_each_entry(reloc, &sec->reloc_list, list) { 857 switch (reloc->sym->type) { 858 case STT_FUNC: 859 func = reloc->sym; 860 break; 861 862 case STT_SECTION: 863 func = find_func_by_offset(reloc->sym->sec, reloc->addend); 864 if (!func) 865 continue; 866 break; 867 868 default: 869 WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type); 870 continue; 871 } 872 873 func_for_each_insn(file, func, insn) 874 insn->ignore = true; 875 } 876 } 877 878 /* 879 * This is a whitelist of functions that is allowed to be called with AC set. 880 * The list is meant to be minimal and only contains compiler instrumentation 881 * ABI and a few functions used to implement *_{to,from}_user() functions. 882 * 883 * These functions must not directly change AC, but may PUSHF/POPF. 884 */ 885 static const char *uaccess_safe_builtin[] = { 886 /* KASAN */ 887 "kasan_report", 888 "kasan_check_range", 889 /* KASAN out-of-line */ 890 "__asan_loadN_noabort", 891 "__asan_load1_noabort", 892 "__asan_load2_noabort", 893 "__asan_load4_noabort", 894 "__asan_load8_noabort", 895 "__asan_load16_noabort", 896 "__asan_storeN_noabort", 897 "__asan_store1_noabort", 898 "__asan_store2_noabort", 899 "__asan_store4_noabort", 900 "__asan_store8_noabort", 901 "__asan_store16_noabort", 902 "__kasan_check_read", 903 "__kasan_check_write", 904 /* KASAN in-line */ 905 "__asan_report_load_n_noabort", 906 "__asan_report_load1_noabort", 907 "__asan_report_load2_noabort", 908 "__asan_report_load4_noabort", 909 "__asan_report_load8_noabort", 910 "__asan_report_load16_noabort", 911 "__asan_report_store_n_noabort", 912 "__asan_report_store1_noabort", 913 "__asan_report_store2_noabort", 914 "__asan_report_store4_noabort", 915 "__asan_report_store8_noabort", 916 "__asan_report_store16_noabort", 917 /* KCSAN */ 918 "__kcsan_check_access", 919 "__kcsan_mb", 920 "__kcsan_wmb", 921 "__kcsan_rmb", 922 "__kcsan_release", 923 "kcsan_found_watchpoint", 924 "kcsan_setup_watchpoint", 925 "kcsan_check_scoped_accesses", 926 "kcsan_disable_current", 927 "kcsan_enable_current_nowarn", 928 /* KCSAN/TSAN */ 929 "__tsan_func_entry", 930 "__tsan_func_exit", 931 "__tsan_read_range", 932 "__tsan_write_range", 933 "__tsan_read1", 934 "__tsan_read2", 935 "__tsan_read4", 936 "__tsan_read8", 937 "__tsan_read16", 938 "__tsan_write1", 939 "__tsan_write2", 940 "__tsan_write4", 941 "__tsan_write8", 942 "__tsan_write16", 943 "__tsan_read_write1", 944 "__tsan_read_write2", 945 "__tsan_read_write4", 946 "__tsan_read_write8", 947 "__tsan_read_write16", 948 "__tsan_atomic8_load", 949 "__tsan_atomic16_load", 950 "__tsan_atomic32_load", 951 "__tsan_atomic64_load", 952 "__tsan_atomic8_store", 953 "__tsan_atomic16_store", 954 "__tsan_atomic32_store", 955 "__tsan_atomic64_store", 956 "__tsan_atomic8_exchange", 957 "__tsan_atomic16_exchange", 958 "__tsan_atomic32_exchange", 959 "__tsan_atomic64_exchange", 960 "__tsan_atomic8_fetch_add", 961 "__tsan_atomic16_fetch_add", 962 "__tsan_atomic32_fetch_add", 963 "__tsan_atomic64_fetch_add", 964 "__tsan_atomic8_fetch_sub", 965 "__tsan_atomic16_fetch_sub", 966 "__tsan_atomic32_fetch_sub", 967 "__tsan_atomic64_fetch_sub", 968 "__tsan_atomic8_fetch_and", 969 "__tsan_atomic16_fetch_and", 970 "__tsan_atomic32_fetch_and", 971 "__tsan_atomic64_fetch_and", 972 "__tsan_atomic8_fetch_or", 973 "__tsan_atomic16_fetch_or", 974 "__tsan_atomic32_fetch_or", 975 "__tsan_atomic64_fetch_or", 976 "__tsan_atomic8_fetch_xor", 977 "__tsan_atomic16_fetch_xor", 978 "__tsan_atomic32_fetch_xor", 979 "__tsan_atomic64_fetch_xor", 980 "__tsan_atomic8_fetch_nand", 981 "__tsan_atomic16_fetch_nand", 982 "__tsan_atomic32_fetch_nand", 983 "__tsan_atomic64_fetch_nand", 984 "__tsan_atomic8_compare_exchange_strong", 985 "__tsan_atomic16_compare_exchange_strong", 986 "__tsan_atomic32_compare_exchange_strong", 987 "__tsan_atomic64_compare_exchange_strong", 988 "__tsan_atomic8_compare_exchange_weak", 989 "__tsan_atomic16_compare_exchange_weak", 990 "__tsan_atomic32_compare_exchange_weak", 991 "__tsan_atomic64_compare_exchange_weak", 992 "__tsan_atomic8_compare_exchange_val", 993 "__tsan_atomic16_compare_exchange_val", 994 "__tsan_atomic32_compare_exchange_val", 995 "__tsan_atomic64_compare_exchange_val", 996 "__tsan_atomic_thread_fence", 997 "__tsan_atomic_signal_fence", 998 /* KCOV */ 999 "write_comp_data", 1000 "check_kcov_mode", 1001 "__sanitizer_cov_trace_pc", 1002 "__sanitizer_cov_trace_const_cmp1", 1003 "__sanitizer_cov_trace_const_cmp2", 1004 "__sanitizer_cov_trace_const_cmp4", 1005 "__sanitizer_cov_trace_const_cmp8", 1006 "__sanitizer_cov_trace_cmp1", 1007 "__sanitizer_cov_trace_cmp2", 1008 "__sanitizer_cov_trace_cmp4", 1009 "__sanitizer_cov_trace_cmp8", 1010 "__sanitizer_cov_trace_switch", 1011 /* UBSAN */ 1012 "ubsan_type_mismatch_common", 1013 "__ubsan_handle_type_mismatch", 1014 "__ubsan_handle_type_mismatch_v1", 1015 "__ubsan_handle_shift_out_of_bounds", 1016 /* misc */ 1017 "csum_partial_copy_generic", 1018 "copy_mc_fragile", 1019 "copy_mc_fragile_handle_tail", 1020 "copy_mc_enhanced_fast_string", 1021 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */ 1022 NULL 1023 }; 1024 1025 static void add_uaccess_safe(struct objtool_file *file) 1026 { 1027 struct symbol *func; 1028 const char **name; 1029 1030 if (!uaccess) 1031 return; 1032 1033 for (name = uaccess_safe_builtin; *name; name++) { 1034 func = find_symbol_by_name(file->elf, *name); 1035 if (!func) 1036 continue; 1037 1038 func->uaccess_safe = true; 1039 } 1040 } 1041 1042 /* 1043 * FIXME: For now, just ignore any alternatives which add retpolines. This is 1044 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline. 1045 * But it at least allows objtool to understand the control flow *around* the 1046 * retpoline. 1047 */ 1048 static int add_ignore_alternatives(struct objtool_file *file) 1049 { 1050 struct section *sec; 1051 struct reloc *reloc; 1052 struct instruction *insn; 1053 1054 sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts"); 1055 if (!sec) 1056 return 0; 1057 1058 list_for_each_entry(reloc, &sec->reloc_list, list) { 1059 if (reloc->sym->type != STT_SECTION) { 1060 WARN("unexpected relocation symbol type in %s", sec->name); 1061 return -1; 1062 } 1063 1064 insn = find_insn(file, reloc->sym->sec, reloc->addend); 1065 if (!insn) { 1066 WARN("bad .discard.ignore_alts entry"); 1067 return -1; 1068 } 1069 1070 insn->ignore_alts = true; 1071 } 1072 1073 return 0; 1074 } 1075 1076 __weak bool arch_is_retpoline(struct symbol *sym) 1077 { 1078 return false; 1079 } 1080 1081 #define NEGATIVE_RELOC ((void *)-1L) 1082 1083 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) 1084 { 1085 if (insn->reloc == NEGATIVE_RELOC) 1086 return NULL; 1087 1088 if (!insn->reloc) { 1089 if (!file) 1090 return NULL; 1091 1092 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec, 1093 insn->offset, insn->len); 1094 if (!insn->reloc) { 1095 insn->reloc = NEGATIVE_RELOC; 1096 return NULL; 1097 } 1098 } 1099 1100 return insn->reloc; 1101 } 1102 1103 static void remove_insn_ops(struct instruction *insn) 1104 { 1105 struct stack_op *op, *tmp; 1106 1107 list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) { 1108 list_del(&op->list); 1109 free(op); 1110 } 1111 } 1112 1113 static void annotate_call_site(struct objtool_file *file, 1114 struct instruction *insn, bool sibling) 1115 { 1116 struct reloc *reloc = insn_reloc(file, insn); 1117 struct symbol *sym = insn->call_dest; 1118 1119 if (!sym) 1120 sym = reloc->sym; 1121 1122 /* 1123 * Alternative replacement code is just template code which is 1124 * sometimes copied to the original instruction. For now, don't 1125 * annotate it. (In the future we might consider annotating the 1126 * original instruction if/when it ever makes sense to do so.) 1127 */ 1128 if (!strcmp(insn->sec->name, ".altinstr_replacement")) 1129 return; 1130 1131 if (sym->static_call_tramp) { 1132 list_add_tail(&insn->call_node, &file->static_call_list); 1133 return; 1134 } 1135 1136 if (sym->retpoline_thunk) { 1137 list_add_tail(&insn->call_node, &file->retpoline_call_list); 1138 return; 1139 } 1140 1141 /* 1142 * Many compilers cannot disable KCOV or sanitizer calls with a function 1143 * attribute so they need a little help, NOP out any such calls from 1144 * noinstr text. 1145 */ 1146 if (insn->sec->noinstr && sym->profiling_func) { 1147 if (reloc) { 1148 reloc->type = R_NONE; 1149 elf_write_reloc(file->elf, reloc); 1150 } 1151 1152 elf_write_insn(file->elf, insn->sec, 1153 insn->offset, insn->len, 1154 sibling ? arch_ret_insn(insn->len) 1155 : arch_nop_insn(insn->len)); 1156 1157 insn->type = sibling ? INSN_RETURN : INSN_NOP; 1158 return; 1159 } 1160 1161 if (mcount && sym->fentry) { 1162 if (sibling) 1163 WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset); 1164 1165 if (reloc) { 1166 reloc->type = R_NONE; 1167 elf_write_reloc(file->elf, reloc); 1168 } 1169 1170 elf_write_insn(file->elf, insn->sec, 1171 insn->offset, insn->len, 1172 arch_nop_insn(insn->len)); 1173 1174 insn->type = INSN_NOP; 1175 1176 list_add_tail(&insn->call_node, &file->mcount_loc_list); 1177 return; 1178 } 1179 1180 if (!sibling && dead_end_function(file, sym)) 1181 insn->dead_end = true; 1182 } 1183 1184 static void add_call_dest(struct objtool_file *file, struct instruction *insn, 1185 struct symbol *dest, bool sibling) 1186 { 1187 insn->call_dest = dest; 1188 if (!dest) 1189 return; 1190 1191 /* 1192 * Whatever stack impact regular CALLs have, should be undone 1193 * by the RETURN of the called function. 1194 * 1195 * Annotated intra-function calls retain the stack_ops but 1196 * are converted to JUMP, see read_intra_function_calls(). 1197 */ 1198 remove_insn_ops(insn); 1199 1200 annotate_call_site(file, insn, sibling); 1201 } 1202 1203 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn) 1204 { 1205 /* 1206 * Retpoline calls/jumps are really dynamic calls/jumps in disguise, 1207 * so convert them accordingly. 1208 */ 1209 switch (insn->type) { 1210 case INSN_CALL: 1211 insn->type = INSN_CALL_DYNAMIC; 1212 break; 1213 case INSN_JUMP_UNCONDITIONAL: 1214 insn->type = INSN_JUMP_DYNAMIC; 1215 break; 1216 case INSN_JUMP_CONDITIONAL: 1217 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL; 1218 break; 1219 default: 1220 return; 1221 } 1222 1223 insn->retpoline_safe = true; 1224 1225 /* 1226 * Whatever stack impact regular CALLs have, should be undone 1227 * by the RETURN of the called function. 1228 * 1229 * Annotated intra-function calls retain the stack_ops but 1230 * are converted to JUMP, see read_intra_function_calls(). 1231 */ 1232 remove_insn_ops(insn); 1233 1234 annotate_call_site(file, insn, false); 1235 } 1236 1237 static bool same_function(struct instruction *insn1, struct instruction *insn2) 1238 { 1239 return insn1->func->pfunc == insn2->func->pfunc; 1240 } 1241 1242 static bool is_first_func_insn(struct instruction *insn) 1243 { 1244 return insn->offset == insn->func->offset || 1245 (insn->type == INSN_ENDBR && 1246 insn->offset == insn->func->offset + insn->len); 1247 } 1248 1249 /* 1250 * Find the destination instructions for all jumps. 1251 */ 1252 static int add_jump_destinations(struct objtool_file *file) 1253 { 1254 struct instruction *insn; 1255 struct reloc *reloc; 1256 struct section *dest_sec; 1257 unsigned long dest_off; 1258 1259 for_each_insn(file, insn) { 1260 if (!is_static_jump(insn)) 1261 continue; 1262 1263 reloc = insn_reloc(file, insn); 1264 if (!reloc) { 1265 dest_sec = insn->sec; 1266 dest_off = arch_jump_destination(insn); 1267 } else if (reloc->sym->type == STT_SECTION) { 1268 dest_sec = reloc->sym->sec; 1269 dest_off = arch_dest_reloc_offset(reloc->addend); 1270 } else if (reloc->sym->retpoline_thunk) { 1271 add_retpoline_call(file, insn); 1272 continue; 1273 } else if (insn->func) { 1274 /* internal or external sibling call (with reloc) */ 1275 add_call_dest(file, insn, reloc->sym, true); 1276 continue; 1277 } else if (reloc->sym->sec->idx) { 1278 dest_sec = reloc->sym->sec; 1279 dest_off = reloc->sym->sym.st_value + 1280 arch_dest_reloc_offset(reloc->addend); 1281 } else { 1282 /* non-func asm code jumping to another file */ 1283 continue; 1284 } 1285 1286 insn->jump_dest = find_insn(file, dest_sec, dest_off); 1287 if (!insn->jump_dest) { 1288 1289 /* 1290 * This is a special case where an alt instruction 1291 * jumps past the end of the section. These are 1292 * handled later in handle_group_alt(). 1293 */ 1294 if (!strcmp(insn->sec->name, ".altinstr_replacement")) 1295 continue; 1296 1297 WARN_FUNC("can't find jump dest instruction at %s+0x%lx", 1298 insn->sec, insn->offset, dest_sec->name, 1299 dest_off); 1300 return -1; 1301 } 1302 1303 /* 1304 * Cross-function jump. 1305 */ 1306 if (insn->func && insn->jump_dest->func && 1307 insn->func != insn->jump_dest->func) { 1308 1309 /* 1310 * For GCC 8+, create parent/child links for any cold 1311 * subfunctions. This is _mostly_ redundant with a 1312 * similar initialization in read_symbols(). 1313 * 1314 * If a function has aliases, we want the *first* such 1315 * function in the symbol table to be the subfunction's 1316 * parent. In that case we overwrite the 1317 * initialization done in read_symbols(). 1318 * 1319 * However this code can't completely replace the 1320 * read_symbols() code because this doesn't detect the 1321 * case where the parent function's only reference to a 1322 * subfunction is through a jump table. 1323 */ 1324 if (!strstr(insn->func->name, ".cold") && 1325 strstr(insn->jump_dest->func->name, ".cold")) { 1326 insn->func->cfunc = insn->jump_dest->func; 1327 insn->jump_dest->func->pfunc = insn->func; 1328 1329 } else if (!same_function(insn, insn->jump_dest) && 1330 is_first_func_insn(insn->jump_dest)) { 1331 /* internal sibling call (without reloc) */ 1332 add_call_dest(file, insn, insn->jump_dest->func, true); 1333 } 1334 } 1335 } 1336 1337 return 0; 1338 } 1339 1340 static struct symbol *find_call_destination(struct section *sec, unsigned long offset) 1341 { 1342 struct symbol *call_dest; 1343 1344 call_dest = find_func_by_offset(sec, offset); 1345 if (!call_dest) 1346 call_dest = find_symbol_by_offset(sec, offset); 1347 1348 return call_dest; 1349 } 1350 1351 /* 1352 * Find the destination instructions for all calls. 1353 */ 1354 static int add_call_destinations(struct objtool_file *file) 1355 { 1356 struct instruction *insn; 1357 unsigned long dest_off; 1358 struct symbol *dest; 1359 struct reloc *reloc; 1360 1361 for_each_insn(file, insn) { 1362 if (insn->type != INSN_CALL) 1363 continue; 1364 1365 reloc = insn_reloc(file, insn); 1366 if (!reloc) { 1367 dest_off = arch_jump_destination(insn); 1368 dest = find_call_destination(insn->sec, dest_off); 1369 1370 add_call_dest(file, insn, dest, false); 1371 1372 if (insn->ignore) 1373 continue; 1374 1375 if (!insn->call_dest) { 1376 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset); 1377 return -1; 1378 } 1379 1380 if (insn->func && insn->call_dest->type != STT_FUNC) { 1381 WARN_FUNC("unsupported call to non-function", 1382 insn->sec, insn->offset); 1383 return -1; 1384 } 1385 1386 } else if (reloc->sym->type == STT_SECTION) { 1387 dest_off = arch_dest_reloc_offset(reloc->addend); 1388 dest = find_call_destination(reloc->sym->sec, dest_off); 1389 if (!dest) { 1390 WARN_FUNC("can't find call dest symbol at %s+0x%lx", 1391 insn->sec, insn->offset, 1392 reloc->sym->sec->name, 1393 dest_off); 1394 return -1; 1395 } 1396 1397 add_call_dest(file, insn, dest, false); 1398 1399 } else if (reloc->sym->retpoline_thunk) { 1400 add_retpoline_call(file, insn); 1401 1402 } else 1403 add_call_dest(file, insn, reloc->sym, false); 1404 } 1405 1406 return 0; 1407 } 1408 1409 /* 1410 * The .alternatives section requires some extra special care over and above 1411 * other special sections because alternatives are patched in place. 1412 */ 1413 static int handle_group_alt(struct objtool_file *file, 1414 struct special_alt *special_alt, 1415 struct instruction *orig_insn, 1416 struct instruction **new_insn) 1417 { 1418 struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL; 1419 struct alt_group *orig_alt_group, *new_alt_group; 1420 unsigned long dest_off; 1421 1422 1423 orig_alt_group = malloc(sizeof(*orig_alt_group)); 1424 if (!orig_alt_group) { 1425 WARN("malloc failed"); 1426 return -1; 1427 } 1428 orig_alt_group->cfi = calloc(special_alt->orig_len, 1429 sizeof(struct cfi_state *)); 1430 if (!orig_alt_group->cfi) { 1431 WARN("calloc failed"); 1432 return -1; 1433 } 1434 1435 last_orig_insn = NULL; 1436 insn = orig_insn; 1437 sec_for_each_insn_from(file, insn) { 1438 if (insn->offset >= special_alt->orig_off + special_alt->orig_len) 1439 break; 1440 1441 insn->alt_group = orig_alt_group; 1442 last_orig_insn = insn; 1443 } 1444 orig_alt_group->orig_group = NULL; 1445 orig_alt_group->first_insn = orig_insn; 1446 orig_alt_group->last_insn = last_orig_insn; 1447 1448 1449 new_alt_group = malloc(sizeof(*new_alt_group)); 1450 if (!new_alt_group) { 1451 WARN("malloc failed"); 1452 return -1; 1453 } 1454 1455 if (special_alt->new_len < special_alt->orig_len) { 1456 /* 1457 * Insert a fake nop at the end to make the replacement 1458 * alt_group the same size as the original. This is needed to 1459 * allow propagate_alt_cfi() to do its magic. When the last 1460 * instruction affects the stack, the instruction after it (the 1461 * nop) will propagate the new state to the shared CFI array. 1462 */ 1463 nop = malloc(sizeof(*nop)); 1464 if (!nop) { 1465 WARN("malloc failed"); 1466 return -1; 1467 } 1468 memset(nop, 0, sizeof(*nop)); 1469 INIT_LIST_HEAD(&nop->alts); 1470 INIT_LIST_HEAD(&nop->stack_ops); 1471 1472 nop->sec = special_alt->new_sec; 1473 nop->offset = special_alt->new_off + special_alt->new_len; 1474 nop->len = special_alt->orig_len - special_alt->new_len; 1475 nop->type = INSN_NOP; 1476 nop->func = orig_insn->func; 1477 nop->alt_group = new_alt_group; 1478 nop->ignore = orig_insn->ignore_alts; 1479 } 1480 1481 if (!special_alt->new_len) { 1482 *new_insn = nop; 1483 goto end; 1484 } 1485 1486 insn = *new_insn; 1487 sec_for_each_insn_from(file, insn) { 1488 struct reloc *alt_reloc; 1489 1490 if (insn->offset >= special_alt->new_off + special_alt->new_len) 1491 break; 1492 1493 last_new_insn = insn; 1494 1495 insn->ignore = orig_insn->ignore_alts; 1496 insn->func = orig_insn->func; 1497 insn->alt_group = new_alt_group; 1498 1499 /* 1500 * Since alternative replacement code is copy/pasted by the 1501 * kernel after applying relocations, generally such code can't 1502 * have relative-address relocation references to outside the 1503 * .altinstr_replacement section, unless the arch's 1504 * alternatives code can adjust the relative offsets 1505 * accordingly. 1506 */ 1507 alt_reloc = insn_reloc(file, insn); 1508 if (alt_reloc && 1509 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { 1510 1511 WARN_FUNC("unsupported relocation in alternatives section", 1512 insn->sec, insn->offset); 1513 return -1; 1514 } 1515 1516 if (!is_static_jump(insn)) 1517 continue; 1518 1519 if (!insn->immediate) 1520 continue; 1521 1522 dest_off = arch_jump_destination(insn); 1523 if (dest_off == special_alt->new_off + special_alt->new_len) 1524 insn->jump_dest = next_insn_same_sec(file, last_orig_insn); 1525 1526 if (!insn->jump_dest) { 1527 WARN_FUNC("can't find alternative jump destination", 1528 insn->sec, insn->offset); 1529 return -1; 1530 } 1531 } 1532 1533 if (!last_new_insn) { 1534 WARN_FUNC("can't find last new alternative instruction", 1535 special_alt->new_sec, special_alt->new_off); 1536 return -1; 1537 } 1538 1539 if (nop) 1540 list_add(&nop->list, &last_new_insn->list); 1541 end: 1542 new_alt_group->orig_group = orig_alt_group; 1543 new_alt_group->first_insn = *new_insn; 1544 new_alt_group->last_insn = nop ? : last_new_insn; 1545 new_alt_group->cfi = orig_alt_group->cfi; 1546 return 0; 1547 } 1548 1549 /* 1550 * A jump table entry can either convert a nop to a jump or a jump to a nop. 1551 * If the original instruction is a jump, make the alt entry an effective nop 1552 * by just skipping the original instruction. 1553 */ 1554 static int handle_jump_alt(struct objtool_file *file, 1555 struct special_alt *special_alt, 1556 struct instruction *orig_insn, 1557 struct instruction **new_insn) 1558 { 1559 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && 1560 orig_insn->type != INSN_NOP) { 1561 1562 WARN_FUNC("unsupported instruction at jump label", 1563 orig_insn->sec, orig_insn->offset); 1564 return -1; 1565 } 1566 1567 if (special_alt->key_addend & 2) { 1568 struct reloc *reloc = insn_reloc(file, orig_insn); 1569 1570 if (reloc) { 1571 reloc->type = R_NONE; 1572 elf_write_reloc(file->elf, reloc); 1573 } 1574 elf_write_insn(file->elf, orig_insn->sec, 1575 orig_insn->offset, orig_insn->len, 1576 arch_nop_insn(orig_insn->len)); 1577 orig_insn->type = INSN_NOP; 1578 } 1579 1580 if (orig_insn->type == INSN_NOP) { 1581 if (orig_insn->len == 2) 1582 file->jl_nop_short++; 1583 else 1584 file->jl_nop_long++; 1585 1586 return 0; 1587 } 1588 1589 if (orig_insn->len == 2) 1590 file->jl_short++; 1591 else 1592 file->jl_long++; 1593 1594 *new_insn = list_next_entry(orig_insn, list); 1595 return 0; 1596 } 1597 1598 /* 1599 * Read all the special sections which have alternate instructions which can be 1600 * patched in or redirected to at runtime. Each instruction having alternate 1601 * instruction(s) has them added to its insn->alts list, which will be 1602 * traversed in validate_branch(). 1603 */ 1604 static int add_special_section_alts(struct objtool_file *file) 1605 { 1606 struct list_head special_alts; 1607 struct instruction *orig_insn, *new_insn; 1608 struct special_alt *special_alt, *tmp; 1609 struct alternative *alt; 1610 int ret; 1611 1612 ret = special_get_alts(file->elf, &special_alts); 1613 if (ret) 1614 return ret; 1615 1616 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) { 1617 1618 orig_insn = find_insn(file, special_alt->orig_sec, 1619 special_alt->orig_off); 1620 if (!orig_insn) { 1621 WARN_FUNC("special: can't find orig instruction", 1622 special_alt->orig_sec, special_alt->orig_off); 1623 ret = -1; 1624 goto out; 1625 } 1626 1627 new_insn = NULL; 1628 if (!special_alt->group || special_alt->new_len) { 1629 new_insn = find_insn(file, special_alt->new_sec, 1630 special_alt->new_off); 1631 if (!new_insn) { 1632 WARN_FUNC("special: can't find new instruction", 1633 special_alt->new_sec, 1634 special_alt->new_off); 1635 ret = -1; 1636 goto out; 1637 } 1638 } 1639 1640 if (special_alt->group) { 1641 if (!special_alt->orig_len) { 1642 WARN_FUNC("empty alternative entry", 1643 orig_insn->sec, orig_insn->offset); 1644 continue; 1645 } 1646 1647 ret = handle_group_alt(file, special_alt, orig_insn, 1648 &new_insn); 1649 if (ret) 1650 goto out; 1651 } else if (special_alt->jump_or_nop) { 1652 ret = handle_jump_alt(file, special_alt, orig_insn, 1653 &new_insn); 1654 if (ret) 1655 goto out; 1656 } 1657 1658 alt = malloc(sizeof(*alt)); 1659 if (!alt) { 1660 WARN("malloc failed"); 1661 ret = -1; 1662 goto out; 1663 } 1664 1665 alt->insn = new_insn; 1666 alt->skip_orig = special_alt->skip_orig; 1667 orig_insn->ignore_alts |= special_alt->skip_alt; 1668 list_add_tail(&alt->list, &orig_insn->alts); 1669 1670 list_del(&special_alt->list); 1671 free(special_alt); 1672 } 1673 1674 if (stats) { 1675 printf("jl\\\tNOP\tJMP\n"); 1676 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); 1677 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); 1678 } 1679 1680 out: 1681 return ret; 1682 } 1683 1684 static int add_jump_table(struct objtool_file *file, struct instruction *insn, 1685 struct reloc *table) 1686 { 1687 struct reloc *reloc = table; 1688 struct instruction *dest_insn; 1689 struct alternative *alt; 1690 struct symbol *pfunc = insn->func->pfunc; 1691 unsigned int prev_offset = 0; 1692 1693 /* 1694 * Each @reloc is a switch table relocation which points to the target 1695 * instruction. 1696 */ 1697 list_for_each_entry_from(reloc, &table->sec->reloc_list, list) { 1698 1699 /* Check for the end of the table: */ 1700 if (reloc != table && reloc->jump_table_start) 1701 break; 1702 1703 /* Make sure the table entries are consecutive: */ 1704 if (prev_offset && reloc->offset != prev_offset + 8) 1705 break; 1706 1707 /* Detect function pointers from contiguous objects: */ 1708 if (reloc->sym->sec == pfunc->sec && 1709 reloc->addend == pfunc->offset) 1710 break; 1711 1712 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend); 1713 if (!dest_insn) 1714 break; 1715 1716 /* Make sure the destination is in the same function: */ 1717 if (!dest_insn->func || dest_insn->func->pfunc != pfunc) 1718 break; 1719 1720 alt = malloc(sizeof(*alt)); 1721 if (!alt) { 1722 WARN("malloc failed"); 1723 return -1; 1724 } 1725 1726 alt->insn = dest_insn; 1727 list_add_tail(&alt->list, &insn->alts); 1728 prev_offset = reloc->offset; 1729 } 1730 1731 if (!prev_offset) { 1732 WARN_FUNC("can't find switch jump table", 1733 insn->sec, insn->offset); 1734 return -1; 1735 } 1736 1737 return 0; 1738 } 1739 1740 /* 1741 * find_jump_table() - Given a dynamic jump, find the switch jump table 1742 * associated with it. 1743 */ 1744 static struct reloc *find_jump_table(struct objtool_file *file, 1745 struct symbol *func, 1746 struct instruction *insn) 1747 { 1748 struct reloc *table_reloc; 1749 struct instruction *dest_insn, *orig_insn = insn; 1750 1751 /* 1752 * Backward search using the @first_jump_src links, these help avoid 1753 * much of the 'in between' code. Which avoids us getting confused by 1754 * it. 1755 */ 1756 for (; 1757 insn && insn->func && insn->func->pfunc == func; 1758 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) { 1759 1760 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC) 1761 break; 1762 1763 /* allow small jumps within the range */ 1764 if (insn->type == INSN_JUMP_UNCONDITIONAL && 1765 insn->jump_dest && 1766 (insn->jump_dest->offset <= insn->offset || 1767 insn->jump_dest->offset > orig_insn->offset)) 1768 break; 1769 1770 table_reloc = arch_find_switch_table(file, insn); 1771 if (!table_reloc) 1772 continue; 1773 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend); 1774 if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func) 1775 continue; 1776 1777 return table_reloc; 1778 } 1779 1780 return NULL; 1781 } 1782 1783 /* 1784 * First pass: Mark the head of each jump table so that in the next pass, 1785 * we know when a given jump table ends and the next one starts. 1786 */ 1787 static void mark_func_jump_tables(struct objtool_file *file, 1788 struct symbol *func) 1789 { 1790 struct instruction *insn, *last = NULL; 1791 struct reloc *reloc; 1792 1793 func_for_each_insn(file, func, insn) { 1794 if (!last) 1795 last = insn; 1796 1797 /* 1798 * Store back-pointers for unconditional forward jumps such 1799 * that find_jump_table() can back-track using those and 1800 * avoid some potentially confusing code. 1801 */ 1802 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && 1803 insn->offset > last->offset && 1804 insn->jump_dest->offset > insn->offset && 1805 !insn->jump_dest->first_jump_src) { 1806 1807 insn->jump_dest->first_jump_src = insn; 1808 last = insn->jump_dest; 1809 } 1810 1811 if (insn->type != INSN_JUMP_DYNAMIC) 1812 continue; 1813 1814 reloc = find_jump_table(file, func, insn); 1815 if (reloc) { 1816 reloc->jump_table_start = true; 1817 insn->jump_table = reloc; 1818 } 1819 } 1820 } 1821 1822 static int add_func_jump_tables(struct objtool_file *file, 1823 struct symbol *func) 1824 { 1825 struct instruction *insn; 1826 int ret; 1827 1828 func_for_each_insn(file, func, insn) { 1829 if (!insn->jump_table) 1830 continue; 1831 1832 ret = add_jump_table(file, insn, insn->jump_table); 1833 if (ret) 1834 return ret; 1835 } 1836 1837 return 0; 1838 } 1839 1840 /* 1841 * For some switch statements, gcc generates a jump table in the .rodata 1842 * section which contains a list of addresses within the function to jump to. 1843 * This finds these jump tables and adds them to the insn->alts lists. 1844 */ 1845 static int add_jump_table_alts(struct objtool_file *file) 1846 { 1847 struct section *sec; 1848 struct symbol *func; 1849 int ret; 1850 1851 if (!file->rodata) 1852 return 0; 1853 1854 for_each_sec(file, sec) { 1855 list_for_each_entry(func, &sec->symbol_list, list) { 1856 if (func->type != STT_FUNC) 1857 continue; 1858 1859 mark_func_jump_tables(file, func); 1860 ret = add_func_jump_tables(file, func); 1861 if (ret) 1862 return ret; 1863 } 1864 } 1865 1866 return 0; 1867 } 1868 1869 static void set_func_state(struct cfi_state *state) 1870 { 1871 state->cfa = initial_func_cfi.cfa; 1872 memcpy(&state->regs, &initial_func_cfi.regs, 1873 CFI_NUM_REGS * sizeof(struct cfi_reg)); 1874 state->stack_size = initial_func_cfi.cfa.offset; 1875 } 1876 1877 static int read_unwind_hints(struct objtool_file *file) 1878 { 1879 struct cfi_state cfi = init_cfi; 1880 struct section *sec, *relocsec; 1881 struct unwind_hint *hint; 1882 struct instruction *insn; 1883 struct reloc *reloc; 1884 int i; 1885 1886 sec = find_section_by_name(file->elf, ".discard.unwind_hints"); 1887 if (!sec) 1888 return 0; 1889 1890 relocsec = sec->reloc; 1891 if (!relocsec) { 1892 WARN("missing .rela.discard.unwind_hints section"); 1893 return -1; 1894 } 1895 1896 if (sec->sh.sh_size % sizeof(struct unwind_hint)) { 1897 WARN("struct unwind_hint size mismatch"); 1898 return -1; 1899 } 1900 1901 file->hints = true; 1902 1903 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) { 1904 hint = (struct unwind_hint *)sec->data->d_buf + i; 1905 1906 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); 1907 if (!reloc) { 1908 WARN("can't find reloc for unwind_hints[%d]", i); 1909 return -1; 1910 } 1911 1912 insn = find_insn(file, reloc->sym->sec, reloc->addend); 1913 if (!insn) { 1914 WARN("can't find insn for unwind_hints[%d]", i); 1915 return -1; 1916 } 1917 1918 insn->hint = true; 1919 1920 if (ibt && hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 1921 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 1922 1923 if (sym && sym->bind == STB_GLOBAL && 1924 insn->type != INSN_ENDBR && !insn->noendbr) { 1925 WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR", 1926 insn->sec, insn->offset); 1927 } 1928 } 1929 1930 if (hint->type == UNWIND_HINT_TYPE_FUNC) { 1931 insn->cfi = &func_cfi; 1932 continue; 1933 } 1934 1935 if (insn->cfi) 1936 cfi = *(insn->cfi); 1937 1938 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { 1939 WARN_FUNC("unsupported unwind_hint sp base reg %d", 1940 insn->sec, insn->offset, hint->sp_reg); 1941 return -1; 1942 } 1943 1944 cfi.cfa.offset = bswap_if_needed(hint->sp_offset); 1945 cfi.type = hint->type; 1946 cfi.end = hint->end; 1947 1948 insn->cfi = cfi_hash_find_or_add(&cfi); 1949 } 1950 1951 return 0; 1952 } 1953 1954 static int read_noendbr_hints(struct objtool_file *file) 1955 { 1956 struct section *sec; 1957 struct instruction *insn; 1958 struct reloc *reloc; 1959 1960 sec = find_section_by_name(file->elf, ".rela.discard.noendbr"); 1961 if (!sec) 1962 return 0; 1963 1964 list_for_each_entry(reloc, &sec->reloc_list, list) { 1965 insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend); 1966 if (!insn) { 1967 WARN("bad .discard.noendbr entry"); 1968 return -1; 1969 } 1970 1971 if (insn->type == INSN_ENDBR) 1972 WARN_FUNC("ANNOTATE_NOENDBR on ENDBR", insn->sec, insn->offset); 1973 1974 insn->noendbr = 1; 1975 } 1976 1977 return 0; 1978 } 1979 1980 static int read_retpoline_hints(struct objtool_file *file) 1981 { 1982 struct section *sec; 1983 struct instruction *insn; 1984 struct reloc *reloc; 1985 1986 sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe"); 1987 if (!sec) 1988 return 0; 1989 1990 list_for_each_entry(reloc, &sec->reloc_list, list) { 1991 if (reloc->sym->type != STT_SECTION) { 1992 WARN("unexpected relocation symbol type in %s", sec->name); 1993 return -1; 1994 } 1995 1996 insn = find_insn(file, reloc->sym->sec, reloc->addend); 1997 if (!insn) { 1998 WARN("bad .discard.retpoline_safe entry"); 1999 return -1; 2000 } 2001 2002 if (insn->type != INSN_JUMP_DYNAMIC && 2003 insn->type != INSN_CALL_DYNAMIC) { 2004 WARN_FUNC("retpoline_safe hint not an indirect jump/call", 2005 insn->sec, insn->offset); 2006 return -1; 2007 } 2008 2009 insn->retpoline_safe = true; 2010 } 2011 2012 return 0; 2013 } 2014 2015 static int read_instr_hints(struct objtool_file *file) 2016 { 2017 struct section *sec; 2018 struct instruction *insn; 2019 struct reloc *reloc; 2020 2021 sec = find_section_by_name(file->elf, ".rela.discard.instr_end"); 2022 if (!sec) 2023 return 0; 2024 2025 list_for_each_entry(reloc, &sec->reloc_list, list) { 2026 if (reloc->sym->type != STT_SECTION) { 2027 WARN("unexpected relocation symbol type in %s", sec->name); 2028 return -1; 2029 } 2030 2031 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2032 if (!insn) { 2033 WARN("bad .discard.instr_end entry"); 2034 return -1; 2035 } 2036 2037 insn->instr--; 2038 } 2039 2040 sec = find_section_by_name(file->elf, ".rela.discard.instr_begin"); 2041 if (!sec) 2042 return 0; 2043 2044 list_for_each_entry(reloc, &sec->reloc_list, list) { 2045 if (reloc->sym->type != STT_SECTION) { 2046 WARN("unexpected relocation symbol type in %s", sec->name); 2047 return -1; 2048 } 2049 2050 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2051 if (!insn) { 2052 WARN("bad .discard.instr_begin entry"); 2053 return -1; 2054 } 2055 2056 insn->instr++; 2057 } 2058 2059 return 0; 2060 } 2061 2062 static int read_intra_function_calls(struct objtool_file *file) 2063 { 2064 struct instruction *insn; 2065 struct section *sec; 2066 struct reloc *reloc; 2067 2068 sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls"); 2069 if (!sec) 2070 return 0; 2071 2072 list_for_each_entry(reloc, &sec->reloc_list, list) { 2073 unsigned long dest_off; 2074 2075 if (reloc->sym->type != STT_SECTION) { 2076 WARN("unexpected relocation symbol type in %s", 2077 sec->name); 2078 return -1; 2079 } 2080 2081 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2082 if (!insn) { 2083 WARN("bad .discard.intra_function_call entry"); 2084 return -1; 2085 } 2086 2087 if (insn->type != INSN_CALL) { 2088 WARN_FUNC("intra_function_call not a direct call", 2089 insn->sec, insn->offset); 2090 return -1; 2091 } 2092 2093 /* 2094 * Treat intra-function CALLs as JMPs, but with a stack_op. 2095 * See add_call_destinations(), which strips stack_ops from 2096 * normal CALLs. 2097 */ 2098 insn->type = INSN_JUMP_UNCONDITIONAL; 2099 2100 dest_off = insn->offset + insn->len + insn->immediate; 2101 insn->jump_dest = find_insn(file, insn->sec, dest_off); 2102 if (!insn->jump_dest) { 2103 WARN_FUNC("can't find call dest at %s+0x%lx", 2104 insn->sec, insn->offset, 2105 insn->sec->name, dest_off); 2106 return -1; 2107 } 2108 } 2109 2110 return 0; 2111 } 2112 2113 /* 2114 * Return true if name matches an instrumentation function, where calls to that 2115 * function from noinstr code can safely be removed, but compilers won't do so. 2116 */ 2117 static bool is_profiling_func(const char *name) 2118 { 2119 /* 2120 * Many compilers cannot disable KCOV with a function attribute. 2121 */ 2122 if (!strncmp(name, "__sanitizer_cov_", 16)) 2123 return true; 2124 2125 /* 2126 * Some compilers currently do not remove __tsan_func_entry/exit nor 2127 * __tsan_atomic_signal_fence (used for barrier instrumentation) with 2128 * the __no_sanitize_thread attribute, remove them. Once the kernel's 2129 * minimum Clang version is 14.0, this can be removed. 2130 */ 2131 if (!strncmp(name, "__tsan_func_", 12) || 2132 !strcmp(name, "__tsan_atomic_signal_fence")) 2133 return true; 2134 2135 return false; 2136 } 2137 2138 static int classify_symbols(struct objtool_file *file) 2139 { 2140 struct section *sec; 2141 struct symbol *func; 2142 2143 for_each_sec(file, sec) { 2144 list_for_each_entry(func, &sec->symbol_list, list) { 2145 if (func->bind != STB_GLOBAL) 2146 continue; 2147 2148 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, 2149 strlen(STATIC_CALL_TRAMP_PREFIX_STR))) 2150 func->static_call_tramp = true; 2151 2152 if (arch_is_retpoline(func)) 2153 func->retpoline_thunk = true; 2154 2155 if (!strcmp(func->name, "__fentry__")) 2156 func->fentry = true; 2157 2158 if (is_profiling_func(func->name)) 2159 func->profiling_func = true; 2160 } 2161 } 2162 2163 return 0; 2164 } 2165 2166 static void mark_rodata(struct objtool_file *file) 2167 { 2168 struct section *sec; 2169 bool found = false; 2170 2171 /* 2172 * Search for the following rodata sections, each of which can 2173 * potentially contain jump tables: 2174 * 2175 * - .rodata: can contain GCC switch tables 2176 * - .rodata.<func>: same, if -fdata-sections is being used 2177 * - .rodata..c_jump_table: contains C annotated jump tables 2178 * 2179 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2180 */ 2181 for_each_sec(file, sec) { 2182 if (!strncmp(sec->name, ".rodata", 7) && 2183 !strstr(sec->name, ".str1.")) { 2184 sec->rodata = true; 2185 found = true; 2186 } 2187 } 2188 2189 file->rodata = found; 2190 } 2191 2192 static int decode_sections(struct objtool_file *file) 2193 { 2194 int ret; 2195 2196 mark_rodata(file); 2197 2198 ret = init_pv_ops(file); 2199 if (ret) 2200 return ret; 2201 2202 ret = decode_instructions(file); 2203 if (ret) 2204 return ret; 2205 2206 add_ignores(file); 2207 add_uaccess_safe(file); 2208 2209 ret = add_ignore_alternatives(file); 2210 if (ret) 2211 return ret; 2212 2213 /* 2214 * Must be before read_unwind_hints() since that needs insn->noendbr. 2215 */ 2216 ret = read_noendbr_hints(file); 2217 if (ret) 2218 return ret; 2219 2220 /* 2221 * Must be before add_{jump_call}_destination. 2222 */ 2223 ret = classify_symbols(file); 2224 if (ret) 2225 return ret; 2226 2227 /* 2228 * Must be before add_special_section_alts() as that depends on 2229 * jump_dest being set. 2230 */ 2231 ret = add_jump_destinations(file); 2232 if (ret) 2233 return ret; 2234 2235 ret = add_special_section_alts(file); 2236 if (ret) 2237 return ret; 2238 2239 /* 2240 * Must be before add_call_destination(); it changes INSN_CALL to 2241 * INSN_JUMP. 2242 */ 2243 ret = read_intra_function_calls(file); 2244 if (ret) 2245 return ret; 2246 2247 ret = add_call_destinations(file); 2248 if (ret) 2249 return ret; 2250 2251 /* 2252 * Must be after add_call_destinations() such that it can override 2253 * dead_end_function() marks. 2254 */ 2255 ret = add_dead_ends(file); 2256 if (ret) 2257 return ret; 2258 2259 ret = add_jump_table_alts(file); 2260 if (ret) 2261 return ret; 2262 2263 ret = read_unwind_hints(file); 2264 if (ret) 2265 return ret; 2266 2267 ret = read_retpoline_hints(file); 2268 if (ret) 2269 return ret; 2270 2271 ret = read_instr_hints(file); 2272 if (ret) 2273 return ret; 2274 2275 return 0; 2276 } 2277 2278 static bool is_fentry_call(struct instruction *insn) 2279 { 2280 if (insn->type == INSN_CALL && 2281 insn->call_dest && 2282 insn->call_dest->fentry) 2283 return true; 2284 2285 return false; 2286 } 2287 2288 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state) 2289 { 2290 struct cfi_state *cfi = &state->cfi; 2291 int i; 2292 2293 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap) 2294 return true; 2295 2296 if (cfi->cfa.offset != initial_func_cfi.cfa.offset) 2297 return true; 2298 2299 if (cfi->stack_size != initial_func_cfi.cfa.offset) 2300 return true; 2301 2302 for (i = 0; i < CFI_NUM_REGS; i++) { 2303 if (cfi->regs[i].base != initial_func_cfi.regs[i].base || 2304 cfi->regs[i].offset != initial_func_cfi.regs[i].offset) 2305 return true; 2306 } 2307 2308 return false; 2309 } 2310 2311 static bool check_reg_frame_pos(const struct cfi_reg *reg, 2312 int expected_offset) 2313 { 2314 return reg->base == CFI_CFA && 2315 reg->offset == expected_offset; 2316 } 2317 2318 static bool has_valid_stack_frame(struct insn_state *state) 2319 { 2320 struct cfi_state *cfi = &state->cfi; 2321 2322 if (cfi->cfa.base == CFI_BP && 2323 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) && 2324 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8)) 2325 return true; 2326 2327 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP) 2328 return true; 2329 2330 return false; 2331 } 2332 2333 static int update_cfi_state_regs(struct instruction *insn, 2334 struct cfi_state *cfi, 2335 struct stack_op *op) 2336 { 2337 struct cfi_reg *cfa = &cfi->cfa; 2338 2339 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT) 2340 return 0; 2341 2342 /* push */ 2343 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF) 2344 cfa->offset += 8; 2345 2346 /* pop */ 2347 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF) 2348 cfa->offset -= 8; 2349 2350 /* add immediate to sp */ 2351 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD && 2352 op->dest.reg == CFI_SP && op->src.reg == CFI_SP) 2353 cfa->offset -= op->src.offset; 2354 2355 return 0; 2356 } 2357 2358 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset) 2359 { 2360 if (arch_callee_saved_reg(reg) && 2361 cfi->regs[reg].base == CFI_UNDEFINED) { 2362 cfi->regs[reg].base = base; 2363 cfi->regs[reg].offset = offset; 2364 } 2365 } 2366 2367 static void restore_reg(struct cfi_state *cfi, unsigned char reg) 2368 { 2369 cfi->regs[reg].base = initial_func_cfi.regs[reg].base; 2370 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset; 2371 } 2372 2373 /* 2374 * A note about DRAP stack alignment: 2375 * 2376 * GCC has the concept of a DRAP register, which is used to help keep track of 2377 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP 2378 * register. The typical DRAP pattern is: 2379 * 2380 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10 2381 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp 2382 * 41 ff 72 f8 pushq -0x8(%r10) 2383 * 55 push %rbp 2384 * 48 89 e5 mov %rsp,%rbp 2385 * (more pushes) 2386 * 41 52 push %r10 2387 * ... 2388 * 41 5a pop %r10 2389 * (more pops) 2390 * 5d pop %rbp 2391 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2392 * c3 retq 2393 * 2394 * There are some variations in the epilogues, like: 2395 * 2396 * 5b pop %rbx 2397 * 41 5a pop %r10 2398 * 41 5c pop %r12 2399 * 41 5d pop %r13 2400 * 41 5e pop %r14 2401 * c9 leaveq 2402 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2403 * c3 retq 2404 * 2405 * and: 2406 * 2407 * 4c 8b 55 e8 mov -0x18(%rbp),%r10 2408 * 48 8b 5d e0 mov -0x20(%rbp),%rbx 2409 * 4c 8b 65 f0 mov -0x10(%rbp),%r12 2410 * 4c 8b 6d f8 mov -0x8(%rbp),%r13 2411 * c9 leaveq 2412 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2413 * c3 retq 2414 * 2415 * Sometimes r13 is used as the DRAP register, in which case it's saved and 2416 * restored beforehand: 2417 * 2418 * 41 55 push %r13 2419 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13 2420 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp 2421 * ... 2422 * 49 8d 65 f0 lea -0x10(%r13),%rsp 2423 * 41 5d pop %r13 2424 * c3 retq 2425 */ 2426 static int update_cfi_state(struct instruction *insn, 2427 struct instruction *next_insn, 2428 struct cfi_state *cfi, struct stack_op *op) 2429 { 2430 struct cfi_reg *cfa = &cfi->cfa; 2431 struct cfi_reg *regs = cfi->regs; 2432 2433 /* stack operations don't make sense with an undefined CFA */ 2434 if (cfa->base == CFI_UNDEFINED) { 2435 if (insn->func) { 2436 WARN_FUNC("undefined stack state", insn->sec, insn->offset); 2437 return -1; 2438 } 2439 return 0; 2440 } 2441 2442 if (cfi->type == UNWIND_HINT_TYPE_REGS || 2443 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) 2444 return update_cfi_state_regs(insn, cfi, op); 2445 2446 switch (op->dest.type) { 2447 2448 case OP_DEST_REG: 2449 switch (op->src.type) { 2450 2451 case OP_SRC_REG: 2452 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP && 2453 cfa->base == CFI_SP && 2454 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) { 2455 2456 /* mov %rsp, %rbp */ 2457 cfa->base = op->dest.reg; 2458 cfi->bp_scratch = false; 2459 } 2460 2461 else if (op->src.reg == CFI_SP && 2462 op->dest.reg == CFI_BP && cfi->drap) { 2463 2464 /* drap: mov %rsp, %rbp */ 2465 regs[CFI_BP].base = CFI_BP; 2466 regs[CFI_BP].offset = -cfi->stack_size; 2467 cfi->bp_scratch = false; 2468 } 2469 2470 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2471 2472 /* 2473 * mov %rsp, %reg 2474 * 2475 * This is needed for the rare case where GCC 2476 * does: 2477 * 2478 * mov %rsp, %rax 2479 * ... 2480 * mov %rax, %rsp 2481 */ 2482 cfi->vals[op->dest.reg].base = CFI_CFA; 2483 cfi->vals[op->dest.reg].offset = -cfi->stack_size; 2484 } 2485 2486 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP && 2487 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) { 2488 2489 /* 2490 * mov %rbp, %rsp 2491 * 2492 * Restore the original stack pointer (Clang). 2493 */ 2494 cfi->stack_size = -cfi->regs[CFI_BP].offset; 2495 } 2496 2497 else if (op->dest.reg == cfa->base) { 2498 2499 /* mov %reg, %rsp */ 2500 if (cfa->base == CFI_SP && 2501 cfi->vals[op->src.reg].base == CFI_CFA) { 2502 2503 /* 2504 * This is needed for the rare case 2505 * where GCC does something dumb like: 2506 * 2507 * lea 0x8(%rsp), %rcx 2508 * ... 2509 * mov %rcx, %rsp 2510 */ 2511 cfa->offset = -cfi->vals[op->src.reg].offset; 2512 cfi->stack_size = cfa->offset; 2513 2514 } else if (cfa->base == CFI_SP && 2515 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2516 cfi->vals[op->src.reg].offset == cfa->offset) { 2517 2518 /* 2519 * Stack swizzle: 2520 * 2521 * 1: mov %rsp, (%[tos]) 2522 * 2: mov %[tos], %rsp 2523 * ... 2524 * 3: pop %rsp 2525 * 2526 * Where: 2527 * 2528 * 1 - places a pointer to the previous 2529 * stack at the Top-of-Stack of the 2530 * new stack. 2531 * 2532 * 2 - switches to the new stack. 2533 * 2534 * 3 - pops the Top-of-Stack to restore 2535 * the original stack. 2536 * 2537 * Note: we set base to SP_INDIRECT 2538 * here and preserve offset. Therefore 2539 * when the unwinder reaches ToS it 2540 * will dereference SP and then add the 2541 * offset to find the next frame, IOW: 2542 * (%rsp) + offset. 2543 */ 2544 cfa->base = CFI_SP_INDIRECT; 2545 2546 } else { 2547 cfa->base = CFI_UNDEFINED; 2548 cfa->offset = 0; 2549 } 2550 } 2551 2552 else if (op->dest.reg == CFI_SP && 2553 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2554 cfi->vals[op->src.reg].offset == cfa->offset) { 2555 2556 /* 2557 * The same stack swizzle case 2) as above. But 2558 * because we can't change cfa->base, case 3) 2559 * will become a regular POP. Pretend we're a 2560 * PUSH so things don't go unbalanced. 2561 */ 2562 cfi->stack_size += 8; 2563 } 2564 2565 2566 break; 2567 2568 case OP_SRC_ADD: 2569 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) { 2570 2571 /* add imm, %rsp */ 2572 cfi->stack_size -= op->src.offset; 2573 if (cfa->base == CFI_SP) 2574 cfa->offset -= op->src.offset; 2575 break; 2576 } 2577 2578 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { 2579 2580 /* lea disp(%rbp), %rsp */ 2581 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); 2582 break; 2583 } 2584 2585 if (!cfi->drap && op->src.reg == CFI_SP && 2586 op->dest.reg == CFI_BP && cfa->base == CFI_SP && 2587 check_reg_frame_pos(®s[CFI_BP], -cfa->offset + op->src.offset)) { 2588 2589 /* lea disp(%rsp), %rbp */ 2590 cfa->base = CFI_BP; 2591 cfa->offset -= op->src.offset; 2592 cfi->bp_scratch = false; 2593 break; 2594 } 2595 2596 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2597 2598 /* drap: lea disp(%rsp), %drap */ 2599 cfi->drap_reg = op->dest.reg; 2600 2601 /* 2602 * lea disp(%rsp), %reg 2603 * 2604 * This is needed for the rare case where GCC 2605 * does something dumb like: 2606 * 2607 * lea 0x8(%rsp), %rcx 2608 * ... 2609 * mov %rcx, %rsp 2610 */ 2611 cfi->vals[op->dest.reg].base = CFI_CFA; 2612 cfi->vals[op->dest.reg].offset = \ 2613 -cfi->stack_size + op->src.offset; 2614 2615 break; 2616 } 2617 2618 if (cfi->drap && op->dest.reg == CFI_SP && 2619 op->src.reg == cfi->drap_reg) { 2620 2621 /* drap: lea disp(%drap), %rsp */ 2622 cfa->base = CFI_SP; 2623 cfa->offset = cfi->stack_size = -op->src.offset; 2624 cfi->drap_reg = CFI_UNDEFINED; 2625 cfi->drap = false; 2626 break; 2627 } 2628 2629 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { 2630 WARN_FUNC("unsupported stack register modification", 2631 insn->sec, insn->offset); 2632 return -1; 2633 } 2634 2635 break; 2636 2637 case OP_SRC_AND: 2638 if (op->dest.reg != CFI_SP || 2639 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || 2640 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { 2641 WARN_FUNC("unsupported stack pointer realignment", 2642 insn->sec, insn->offset); 2643 return -1; 2644 } 2645 2646 if (cfi->drap_reg != CFI_UNDEFINED) { 2647 /* drap: and imm, %rsp */ 2648 cfa->base = cfi->drap_reg; 2649 cfa->offset = cfi->stack_size = 0; 2650 cfi->drap = true; 2651 } 2652 2653 /* 2654 * Older versions of GCC (4.8ish) realign the stack 2655 * without DRAP, with a frame pointer. 2656 */ 2657 2658 break; 2659 2660 case OP_SRC_POP: 2661 case OP_SRC_POPF: 2662 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) { 2663 2664 /* pop %rsp; # restore from a stack swizzle */ 2665 cfa->base = CFI_SP; 2666 break; 2667 } 2668 2669 if (!cfi->drap && op->dest.reg == cfa->base) { 2670 2671 /* pop %rbp */ 2672 cfa->base = CFI_SP; 2673 } 2674 2675 if (cfi->drap && cfa->base == CFI_BP_INDIRECT && 2676 op->dest.reg == cfi->drap_reg && 2677 cfi->drap_offset == -cfi->stack_size) { 2678 2679 /* drap: pop %drap */ 2680 cfa->base = cfi->drap_reg; 2681 cfa->offset = 0; 2682 cfi->drap_offset = -1; 2683 2684 } else if (cfi->stack_size == -regs[op->dest.reg].offset) { 2685 2686 /* pop %reg */ 2687 restore_reg(cfi, op->dest.reg); 2688 } 2689 2690 cfi->stack_size -= 8; 2691 if (cfa->base == CFI_SP) 2692 cfa->offset -= 8; 2693 2694 break; 2695 2696 case OP_SRC_REG_INDIRECT: 2697 if (!cfi->drap && op->dest.reg == cfa->base && 2698 op->dest.reg == CFI_BP) { 2699 2700 /* mov disp(%rsp), %rbp */ 2701 cfa->base = CFI_SP; 2702 cfa->offset = cfi->stack_size; 2703 } 2704 2705 if (cfi->drap && op->src.reg == CFI_BP && 2706 op->src.offset == cfi->drap_offset) { 2707 2708 /* drap: mov disp(%rbp), %drap */ 2709 cfa->base = cfi->drap_reg; 2710 cfa->offset = 0; 2711 cfi->drap_offset = -1; 2712 } 2713 2714 if (cfi->drap && op->src.reg == CFI_BP && 2715 op->src.offset == regs[op->dest.reg].offset) { 2716 2717 /* drap: mov disp(%rbp), %reg */ 2718 restore_reg(cfi, op->dest.reg); 2719 2720 } else if (op->src.reg == cfa->base && 2721 op->src.offset == regs[op->dest.reg].offset + cfa->offset) { 2722 2723 /* mov disp(%rbp), %reg */ 2724 /* mov disp(%rsp), %reg */ 2725 restore_reg(cfi, op->dest.reg); 2726 2727 } else if (op->src.reg == CFI_SP && 2728 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) { 2729 2730 /* mov disp(%rsp), %reg */ 2731 restore_reg(cfi, op->dest.reg); 2732 } 2733 2734 break; 2735 2736 default: 2737 WARN_FUNC("unknown stack-related instruction", 2738 insn->sec, insn->offset); 2739 return -1; 2740 } 2741 2742 break; 2743 2744 case OP_DEST_PUSH: 2745 case OP_DEST_PUSHF: 2746 cfi->stack_size += 8; 2747 if (cfa->base == CFI_SP) 2748 cfa->offset += 8; 2749 2750 if (op->src.type != OP_SRC_REG) 2751 break; 2752 2753 if (cfi->drap) { 2754 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 2755 2756 /* drap: push %drap */ 2757 cfa->base = CFI_BP_INDIRECT; 2758 cfa->offset = -cfi->stack_size; 2759 2760 /* save drap so we know when to restore it */ 2761 cfi->drap_offset = -cfi->stack_size; 2762 2763 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) { 2764 2765 /* drap: push %rbp */ 2766 cfi->stack_size = 0; 2767 2768 } else { 2769 2770 /* drap: push %reg */ 2771 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); 2772 } 2773 2774 } else { 2775 2776 /* push %reg */ 2777 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size); 2778 } 2779 2780 /* detect when asm code uses rbp as a scratch register */ 2781 if (!no_fp && insn->func && op->src.reg == CFI_BP && 2782 cfa->base != CFI_BP) 2783 cfi->bp_scratch = true; 2784 break; 2785 2786 case OP_DEST_REG_INDIRECT: 2787 2788 if (cfi->drap) { 2789 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 2790 2791 /* drap: mov %drap, disp(%rbp) */ 2792 cfa->base = CFI_BP_INDIRECT; 2793 cfa->offset = op->dest.offset; 2794 2795 /* save drap offset so we know when to restore it */ 2796 cfi->drap_offset = op->dest.offset; 2797 } else { 2798 2799 /* drap: mov reg, disp(%rbp) */ 2800 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); 2801 } 2802 2803 } else if (op->dest.reg == cfa->base) { 2804 2805 /* mov reg, disp(%rbp) */ 2806 /* mov reg, disp(%rsp) */ 2807 save_reg(cfi, op->src.reg, CFI_CFA, 2808 op->dest.offset - cfi->cfa.offset); 2809 2810 } else if (op->dest.reg == CFI_SP) { 2811 2812 /* mov reg, disp(%rsp) */ 2813 save_reg(cfi, op->src.reg, CFI_CFA, 2814 op->dest.offset - cfi->stack_size); 2815 2816 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) { 2817 2818 /* mov %rsp, (%reg); # setup a stack swizzle. */ 2819 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT; 2820 cfi->vals[op->dest.reg].offset = cfa->offset; 2821 } 2822 2823 break; 2824 2825 case OP_DEST_MEM: 2826 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { 2827 WARN_FUNC("unknown stack-related memory operation", 2828 insn->sec, insn->offset); 2829 return -1; 2830 } 2831 2832 /* pop mem */ 2833 cfi->stack_size -= 8; 2834 if (cfa->base == CFI_SP) 2835 cfa->offset -= 8; 2836 2837 break; 2838 2839 default: 2840 WARN_FUNC("unknown stack-related instruction", 2841 insn->sec, insn->offset); 2842 return -1; 2843 } 2844 2845 return 0; 2846 } 2847 2848 /* 2849 * The stack layouts of alternatives instructions can sometimes diverge when 2850 * they have stack modifications. That's fine as long as the potential stack 2851 * layouts don't conflict at any given potential instruction boundary. 2852 * 2853 * Flatten the CFIs of the different alternative code streams (both original 2854 * and replacement) into a single shared CFI array which can be used to detect 2855 * conflicts and nicely feed a linear array of ORC entries to the unwinder. 2856 */ 2857 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn) 2858 { 2859 struct cfi_state **alt_cfi; 2860 int group_off; 2861 2862 if (!insn->alt_group) 2863 return 0; 2864 2865 if (!insn->cfi) { 2866 WARN("CFI missing"); 2867 return -1; 2868 } 2869 2870 alt_cfi = insn->alt_group->cfi; 2871 group_off = insn->offset - insn->alt_group->first_insn->offset; 2872 2873 if (!alt_cfi[group_off]) { 2874 alt_cfi[group_off] = insn->cfi; 2875 } else { 2876 if (cficmp(alt_cfi[group_off], insn->cfi)) { 2877 WARN_FUNC("stack layout conflict in alternatives", 2878 insn->sec, insn->offset); 2879 return -1; 2880 } 2881 } 2882 2883 return 0; 2884 } 2885 2886 static int handle_insn_ops(struct instruction *insn, 2887 struct instruction *next_insn, 2888 struct insn_state *state) 2889 { 2890 struct stack_op *op; 2891 2892 list_for_each_entry(op, &insn->stack_ops, list) { 2893 2894 if (update_cfi_state(insn, next_insn, &state->cfi, op)) 2895 return 1; 2896 2897 if (!insn->alt_group) 2898 continue; 2899 2900 if (op->dest.type == OP_DEST_PUSHF) { 2901 if (!state->uaccess_stack) { 2902 state->uaccess_stack = 1; 2903 } else if (state->uaccess_stack >> 31) { 2904 WARN_FUNC("PUSHF stack exhausted", 2905 insn->sec, insn->offset); 2906 return 1; 2907 } 2908 state->uaccess_stack <<= 1; 2909 state->uaccess_stack |= state->uaccess; 2910 } 2911 2912 if (op->src.type == OP_SRC_POPF) { 2913 if (state->uaccess_stack) { 2914 state->uaccess = state->uaccess_stack & 1; 2915 state->uaccess_stack >>= 1; 2916 if (state->uaccess_stack == 1) 2917 state->uaccess_stack = 0; 2918 } 2919 } 2920 } 2921 2922 return 0; 2923 } 2924 2925 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) 2926 { 2927 struct cfi_state *cfi1 = insn->cfi; 2928 int i; 2929 2930 if (!cfi1) { 2931 WARN("CFI missing"); 2932 return false; 2933 } 2934 2935 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { 2936 2937 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d", 2938 insn->sec, insn->offset, 2939 cfi1->cfa.base, cfi1->cfa.offset, 2940 cfi2->cfa.base, cfi2->cfa.offset); 2941 2942 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) { 2943 for (i = 0; i < CFI_NUM_REGS; i++) { 2944 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], 2945 sizeof(struct cfi_reg))) 2946 continue; 2947 2948 WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", 2949 insn->sec, insn->offset, 2950 i, cfi1->regs[i].base, cfi1->regs[i].offset, 2951 i, cfi2->regs[i].base, cfi2->regs[i].offset); 2952 break; 2953 } 2954 2955 } else if (cfi1->type != cfi2->type) { 2956 2957 WARN_FUNC("stack state mismatch: type1=%d type2=%d", 2958 insn->sec, insn->offset, cfi1->type, cfi2->type); 2959 2960 } else if (cfi1->drap != cfi2->drap || 2961 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || 2962 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { 2963 2964 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", 2965 insn->sec, insn->offset, 2966 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, 2967 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); 2968 2969 } else 2970 return true; 2971 2972 return false; 2973 } 2974 2975 static inline bool func_uaccess_safe(struct symbol *func) 2976 { 2977 if (func) 2978 return func->uaccess_safe; 2979 2980 return false; 2981 } 2982 2983 static inline const char *call_dest_name(struct instruction *insn) 2984 { 2985 static char pvname[19]; 2986 struct reloc *rel; 2987 int idx; 2988 2989 if (insn->call_dest) 2990 return insn->call_dest->name; 2991 2992 rel = insn_reloc(NULL, insn); 2993 if (rel && !strcmp(rel->sym->name, "pv_ops")) { 2994 idx = (rel->addend / sizeof(void *)); 2995 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx); 2996 return pvname; 2997 } 2998 2999 return "{dynamic}"; 3000 } 3001 3002 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn) 3003 { 3004 struct symbol *target; 3005 struct reloc *rel; 3006 int idx; 3007 3008 rel = insn_reloc(file, insn); 3009 if (!rel || strcmp(rel->sym->name, "pv_ops")) 3010 return false; 3011 3012 idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *)); 3013 3014 if (file->pv_ops[idx].clean) 3015 return true; 3016 3017 file->pv_ops[idx].clean = true; 3018 3019 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) { 3020 if (!target->sec->noinstr) { 3021 WARN("pv_ops[%d]: %s", idx, target->name); 3022 file->pv_ops[idx].clean = false; 3023 } 3024 } 3025 3026 return file->pv_ops[idx].clean; 3027 } 3028 3029 static inline bool noinstr_call_dest(struct objtool_file *file, 3030 struct instruction *insn, 3031 struct symbol *func) 3032 { 3033 /* 3034 * We can't deal with indirect function calls at present; 3035 * assume they're instrumented. 3036 */ 3037 if (!func) { 3038 if (file->pv_ops) 3039 return pv_call_dest(file, insn); 3040 3041 return false; 3042 } 3043 3044 /* 3045 * If the symbol is from a noinstr section; we good. 3046 */ 3047 if (func->sec->noinstr) 3048 return true; 3049 3050 /* 3051 * The __ubsan_handle_*() calls are like WARN(), they only happen when 3052 * something 'BAD' happened. At the risk of taking the machine down, 3053 * let them proceed to get the message out. 3054 */ 3055 if (!strncmp(func->name, "__ubsan_handle_", 15)) 3056 return true; 3057 3058 return false; 3059 } 3060 3061 static int validate_call(struct objtool_file *file, 3062 struct instruction *insn, 3063 struct insn_state *state) 3064 { 3065 if (state->noinstr && state->instr <= 0 && 3066 !noinstr_call_dest(file, insn, insn->call_dest)) { 3067 WARN_FUNC("call to %s() leaves .noinstr.text section", 3068 insn->sec, insn->offset, call_dest_name(insn)); 3069 return 1; 3070 } 3071 3072 if (state->uaccess && !func_uaccess_safe(insn->call_dest)) { 3073 WARN_FUNC("call to %s() with UACCESS enabled", 3074 insn->sec, insn->offset, call_dest_name(insn)); 3075 return 1; 3076 } 3077 3078 if (state->df) { 3079 WARN_FUNC("call to %s() with DF set", 3080 insn->sec, insn->offset, call_dest_name(insn)); 3081 return 1; 3082 } 3083 3084 return 0; 3085 } 3086 3087 static int validate_sibling_call(struct objtool_file *file, 3088 struct instruction *insn, 3089 struct insn_state *state) 3090 { 3091 if (has_modified_stack_frame(insn, state)) { 3092 WARN_FUNC("sibling call from callable instruction with modified stack frame", 3093 insn->sec, insn->offset); 3094 return 1; 3095 } 3096 3097 return validate_call(file, insn, state); 3098 } 3099 3100 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 3101 { 3102 if (state->noinstr && state->instr > 0) { 3103 WARN_FUNC("return with instrumentation enabled", 3104 insn->sec, insn->offset); 3105 return 1; 3106 } 3107 3108 if (state->uaccess && !func_uaccess_safe(func)) { 3109 WARN_FUNC("return with UACCESS enabled", 3110 insn->sec, insn->offset); 3111 return 1; 3112 } 3113 3114 if (!state->uaccess && func_uaccess_safe(func)) { 3115 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function", 3116 insn->sec, insn->offset); 3117 return 1; 3118 } 3119 3120 if (state->df) { 3121 WARN_FUNC("return with DF set", 3122 insn->sec, insn->offset); 3123 return 1; 3124 } 3125 3126 if (func && has_modified_stack_frame(insn, state)) { 3127 WARN_FUNC("return with modified stack frame", 3128 insn->sec, insn->offset); 3129 return 1; 3130 } 3131 3132 if (state->cfi.bp_scratch) { 3133 WARN_FUNC("BP used as a scratch register", 3134 insn->sec, insn->offset); 3135 return 1; 3136 } 3137 3138 return 0; 3139 } 3140 3141 static struct instruction *next_insn_to_validate(struct objtool_file *file, 3142 struct instruction *insn) 3143 { 3144 struct alt_group *alt_group = insn->alt_group; 3145 3146 /* 3147 * Simulate the fact that alternatives are patched in-place. When the 3148 * end of a replacement alt_group is reached, redirect objtool flow to 3149 * the end of the original alt_group. 3150 */ 3151 if (alt_group && insn == alt_group->last_insn && alt_group->orig_group) 3152 return next_insn_same_sec(file, alt_group->orig_group->last_insn); 3153 3154 return next_insn_same_sec(file, insn); 3155 } 3156 3157 static struct instruction * 3158 validate_ibt_reloc(struct objtool_file *file, struct reloc *reloc) 3159 { 3160 struct instruction *dest; 3161 struct section *sec; 3162 unsigned long off; 3163 3164 sec = reloc->sym->sec; 3165 off = reloc->sym->offset; 3166 3167 if ((reloc->sec->base->sh.sh_flags & SHF_EXECINSTR) && 3168 (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)) 3169 off += arch_dest_reloc_offset(reloc->addend); 3170 else 3171 off += reloc->addend; 3172 3173 dest = find_insn(file, sec, off); 3174 if (!dest) 3175 return NULL; 3176 3177 if (dest->type == INSN_ENDBR) { 3178 if (!list_empty(&dest->call_node)) 3179 list_del_init(&dest->call_node); 3180 3181 return NULL; 3182 } 3183 3184 if (reloc->sym->static_call_tramp) 3185 return NULL; 3186 3187 return dest; 3188 } 3189 3190 static void warn_noendbr(const char *msg, struct section *sec, unsigned long offset, 3191 struct instruction *dest) 3192 { 3193 WARN_FUNC("%srelocation to !ENDBR: %s+0x%lx", sec, offset, msg, 3194 dest->func ? dest->func->name : dest->sec->name, 3195 dest->func ? dest->offset - dest->func->offset : dest->offset); 3196 } 3197 3198 static void validate_ibt_dest(struct objtool_file *file, struct instruction *insn, 3199 struct instruction *dest) 3200 { 3201 if (dest->func && dest->func == insn->func) { 3202 /* 3203 * Anything from->to self is either _THIS_IP_ or IRET-to-self. 3204 * 3205 * There is no sane way to annotate _THIS_IP_ since the compiler treats the 3206 * relocation as a constant and is happy to fold in offsets, skewing any 3207 * annotation we do, leading to vast amounts of false-positives. 3208 * 3209 * There's also compiler generated _THIS_IP_ through KCOV and 3210 * such which we have no hope of annotating. 3211 * 3212 * As such, blanket accept self-references without issue. 3213 */ 3214 return; 3215 } 3216 3217 if (dest->noendbr) 3218 return; 3219 3220 warn_noendbr("", insn->sec, insn->offset, dest); 3221 } 3222 3223 static void validate_ibt_insn(struct objtool_file *file, struct instruction *insn) 3224 { 3225 struct instruction *dest; 3226 struct reloc *reloc; 3227 3228 switch (insn->type) { 3229 case INSN_CALL: 3230 case INSN_CALL_DYNAMIC: 3231 case INSN_JUMP_CONDITIONAL: 3232 case INSN_JUMP_UNCONDITIONAL: 3233 case INSN_JUMP_DYNAMIC: 3234 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3235 case INSN_RETURN: 3236 /* 3237 * We're looking for code references setting up indirect code 3238 * flow. As such, ignore direct code flow and the actual 3239 * dynamic branches. 3240 */ 3241 return; 3242 3243 case INSN_NOP: 3244 /* 3245 * handle_group_alt() will create INSN_NOP instruction that 3246 * don't belong to any section, ignore all NOP since they won't 3247 * carry a (useful) relocation anyway. 3248 */ 3249 return; 3250 3251 default: 3252 break; 3253 } 3254 3255 for (reloc = insn_reloc(file, insn); 3256 reloc; 3257 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 3258 reloc->offset + 1, 3259 (insn->offset + insn->len) - (reloc->offset + 1))) { 3260 dest = validate_ibt_reloc(file, reloc); 3261 if (dest) 3262 validate_ibt_dest(file, insn, dest); 3263 } 3264 } 3265 3266 /* 3267 * Follow the branch starting at the given instruction, and recursively follow 3268 * any other branches (jumps). Meanwhile, track the frame pointer state at 3269 * each instruction and validate all the rules described in 3270 * tools/objtool/Documentation/stack-validation.txt. 3271 */ 3272 static int validate_branch(struct objtool_file *file, struct symbol *func, 3273 struct instruction *insn, struct insn_state state) 3274 { 3275 struct alternative *alt; 3276 struct instruction *next_insn, *prev_insn = NULL; 3277 struct section *sec; 3278 u8 visited; 3279 int ret; 3280 3281 sec = insn->sec; 3282 3283 while (1) { 3284 next_insn = next_insn_to_validate(file, insn); 3285 3286 if (file->c_file && func && insn->func && func != insn->func->pfunc) { 3287 WARN("%s() falls through to next function %s()", 3288 func->name, insn->func->name); 3289 return 1; 3290 } 3291 3292 if (func && insn->ignore) { 3293 WARN_FUNC("BUG: why am I validating an ignored function?", 3294 sec, insn->offset); 3295 return 1; 3296 } 3297 3298 visited = 1 << state.uaccess; 3299 if (insn->visited) { 3300 if (!insn->hint && !insn_cfi_match(insn, &state.cfi)) 3301 return 1; 3302 3303 if (insn->visited & visited) 3304 return 0; 3305 } else { 3306 nr_insns_visited++; 3307 } 3308 3309 if (state.noinstr) 3310 state.instr += insn->instr; 3311 3312 if (insn->hint) { 3313 state.cfi = *insn->cfi; 3314 } else { 3315 /* XXX track if we actually changed state.cfi */ 3316 3317 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) { 3318 insn->cfi = prev_insn->cfi; 3319 nr_cfi_reused++; 3320 } else { 3321 insn->cfi = cfi_hash_find_or_add(&state.cfi); 3322 } 3323 } 3324 3325 insn->visited |= visited; 3326 3327 if (propagate_alt_cfi(file, insn)) 3328 return 1; 3329 3330 if (!insn->ignore_alts && !list_empty(&insn->alts)) { 3331 bool skip_orig = false; 3332 3333 list_for_each_entry(alt, &insn->alts, list) { 3334 if (alt->skip_orig) 3335 skip_orig = true; 3336 3337 ret = validate_branch(file, func, alt->insn, state); 3338 if (ret) { 3339 if (backtrace) 3340 BT_FUNC("(alt)", insn); 3341 return ret; 3342 } 3343 } 3344 3345 if (skip_orig) 3346 return 0; 3347 } 3348 3349 if (handle_insn_ops(insn, next_insn, &state)) 3350 return 1; 3351 3352 switch (insn->type) { 3353 3354 case INSN_RETURN: 3355 if (sls && !insn->retpoline_safe && 3356 next_insn && next_insn->type != INSN_TRAP) { 3357 WARN_FUNC("missing int3 after ret", 3358 insn->sec, insn->offset); 3359 } 3360 return validate_return(func, insn, &state); 3361 3362 case INSN_CALL: 3363 case INSN_CALL_DYNAMIC: 3364 ret = validate_call(file, insn, &state); 3365 if (ret) 3366 return ret; 3367 3368 if (!no_fp && func && !is_fentry_call(insn) && 3369 !has_valid_stack_frame(&state)) { 3370 WARN_FUNC("call without frame pointer save/setup", 3371 sec, insn->offset); 3372 return 1; 3373 } 3374 3375 if (insn->dead_end) 3376 return 0; 3377 3378 break; 3379 3380 case INSN_JUMP_CONDITIONAL: 3381 case INSN_JUMP_UNCONDITIONAL: 3382 if (is_sibling_call(insn)) { 3383 ret = validate_sibling_call(file, insn, &state); 3384 if (ret) 3385 return ret; 3386 3387 } else if (insn->jump_dest) { 3388 ret = validate_branch(file, func, 3389 insn->jump_dest, state); 3390 if (ret) { 3391 if (backtrace) 3392 BT_FUNC("(branch)", insn); 3393 return ret; 3394 } 3395 } 3396 3397 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3398 return 0; 3399 3400 break; 3401 3402 case INSN_JUMP_DYNAMIC: 3403 if (sls && !insn->retpoline_safe && 3404 next_insn && next_insn->type != INSN_TRAP) { 3405 WARN_FUNC("missing int3 after indirect jump", 3406 insn->sec, insn->offset); 3407 } 3408 3409 /* fallthrough */ 3410 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3411 if (is_sibling_call(insn)) { 3412 ret = validate_sibling_call(file, insn, &state); 3413 if (ret) 3414 return ret; 3415 } 3416 3417 if (insn->type == INSN_JUMP_DYNAMIC) 3418 return 0; 3419 3420 break; 3421 3422 case INSN_CONTEXT_SWITCH: 3423 if (func && (!next_insn || !next_insn->hint)) { 3424 WARN_FUNC("unsupported instruction in callable function", 3425 sec, insn->offset); 3426 return 1; 3427 } 3428 return 0; 3429 3430 case INSN_STAC: 3431 if (state.uaccess) { 3432 WARN_FUNC("recursive UACCESS enable", sec, insn->offset); 3433 return 1; 3434 } 3435 3436 state.uaccess = true; 3437 break; 3438 3439 case INSN_CLAC: 3440 if (!state.uaccess && func) { 3441 WARN_FUNC("redundant UACCESS disable", sec, insn->offset); 3442 return 1; 3443 } 3444 3445 if (func_uaccess_safe(func) && !state.uaccess_stack) { 3446 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset); 3447 return 1; 3448 } 3449 3450 state.uaccess = false; 3451 break; 3452 3453 case INSN_STD: 3454 if (state.df) { 3455 WARN_FUNC("recursive STD", sec, insn->offset); 3456 return 1; 3457 } 3458 3459 state.df = true; 3460 break; 3461 3462 case INSN_CLD: 3463 if (!state.df && func) { 3464 WARN_FUNC("redundant CLD", sec, insn->offset); 3465 return 1; 3466 } 3467 3468 state.df = false; 3469 break; 3470 3471 default: 3472 break; 3473 } 3474 3475 if (ibt) 3476 validate_ibt_insn(file, insn); 3477 3478 if (insn->dead_end) 3479 return 0; 3480 3481 if (!next_insn) { 3482 if (state.cfi.cfa.base == CFI_UNDEFINED) 3483 return 0; 3484 WARN("%s: unexpected end of section", sec->name); 3485 return 1; 3486 } 3487 3488 prev_insn = insn; 3489 insn = next_insn; 3490 } 3491 3492 return 0; 3493 } 3494 3495 static int validate_unwind_hints(struct objtool_file *file, struct section *sec) 3496 { 3497 struct instruction *insn; 3498 struct insn_state state; 3499 int ret, warnings = 0; 3500 3501 if (!file->hints) 3502 return 0; 3503 3504 init_insn_state(&state, sec); 3505 3506 if (sec) { 3507 insn = find_insn(file, sec, 0); 3508 if (!insn) 3509 return 0; 3510 } else { 3511 insn = list_first_entry(&file->insn_list, typeof(*insn), list); 3512 } 3513 3514 while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) { 3515 if (insn->hint && !insn->visited && !insn->ignore) { 3516 ret = validate_branch(file, insn->func, insn, state); 3517 if (ret && backtrace) 3518 BT_FUNC("<=== (hint)", insn); 3519 warnings += ret; 3520 } 3521 3522 insn = list_next_entry(insn, list); 3523 } 3524 3525 return warnings; 3526 } 3527 3528 static int validate_retpoline(struct objtool_file *file) 3529 { 3530 struct instruction *insn; 3531 int warnings = 0; 3532 3533 for_each_insn(file, insn) { 3534 if (insn->type != INSN_JUMP_DYNAMIC && 3535 insn->type != INSN_CALL_DYNAMIC) 3536 continue; 3537 3538 if (insn->retpoline_safe) 3539 continue; 3540 3541 /* 3542 * .init.text code is ran before userspace and thus doesn't 3543 * strictly need retpolines, except for modules which are 3544 * loaded late, they very much do need retpoline in their 3545 * .init.text 3546 */ 3547 if (!strcmp(insn->sec->name, ".init.text") && !module) 3548 continue; 3549 3550 WARN_FUNC("indirect %s found in RETPOLINE build", 3551 insn->sec, insn->offset, 3552 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); 3553 3554 warnings++; 3555 } 3556 3557 return warnings; 3558 } 3559 3560 static bool is_kasan_insn(struct instruction *insn) 3561 { 3562 return (insn->type == INSN_CALL && 3563 !strcmp(insn->call_dest->name, "__asan_handle_no_return")); 3564 } 3565 3566 static bool is_ubsan_insn(struct instruction *insn) 3567 { 3568 return (insn->type == INSN_CALL && 3569 !strcmp(insn->call_dest->name, 3570 "__ubsan_handle_builtin_unreachable")); 3571 } 3572 3573 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) 3574 { 3575 int i; 3576 struct instruction *prev_insn; 3577 3578 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP) 3579 return true; 3580 3581 /* 3582 * Ignore alternative replacement instructions. This can happen 3583 * when a whitelisted function uses one of the ALTERNATIVE macros. 3584 */ 3585 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 3586 !strcmp(insn->sec->name, ".altinstr_aux")) 3587 return true; 3588 3589 /* 3590 * Whole archive runs might encounder dead code from weak symbols. 3591 * This is where the linker will have dropped the weak symbol in 3592 * favour of a regular symbol, but leaves the code in place. 3593 * 3594 * In this case we'll find a piece of code (whole function) that is not 3595 * covered by a !section symbol. Ignore them. 3596 */ 3597 if (!insn->func && lto) { 3598 int size = find_symbol_hole_containing(insn->sec, insn->offset); 3599 unsigned long end = insn->offset + size; 3600 3601 if (!size) /* not a hole */ 3602 return false; 3603 3604 if (size < 0) /* hole until the end */ 3605 return true; 3606 3607 sec_for_each_insn_continue(file, insn) { 3608 /* 3609 * If we reach a visited instruction at or before the 3610 * end of the hole, ignore the unreachable. 3611 */ 3612 if (insn->visited) 3613 return true; 3614 3615 if (insn->offset >= end) 3616 break; 3617 3618 /* 3619 * If this hole jumps to a .cold function, mark it ignore too. 3620 */ 3621 if (insn->jump_dest && insn->jump_dest->func && 3622 strstr(insn->jump_dest->func->name, ".cold")) { 3623 struct instruction *dest = insn->jump_dest; 3624 func_for_each_insn(file, dest->func, dest) 3625 dest->ignore = true; 3626 } 3627 } 3628 3629 return false; 3630 } 3631 3632 if (!insn->func) 3633 return false; 3634 3635 if (insn->func->static_call_tramp) 3636 return true; 3637 3638 /* 3639 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees 3640 * __builtin_unreachable(). The BUG() macro has an unreachable() after 3641 * the UD2, which causes GCC's undefined trap logic to emit another UD2 3642 * (or occasionally a JMP to UD2). 3643 * 3644 * It may also insert a UD2 after calling a __noreturn function. 3645 */ 3646 prev_insn = list_prev_entry(insn, list); 3647 if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) && 3648 (insn->type == INSN_BUG || 3649 (insn->type == INSN_JUMP_UNCONDITIONAL && 3650 insn->jump_dest && insn->jump_dest->type == INSN_BUG))) 3651 return true; 3652 3653 /* 3654 * Check if this (or a subsequent) instruction is related to 3655 * CONFIG_UBSAN or CONFIG_KASAN. 3656 * 3657 * End the search at 5 instructions to avoid going into the weeds. 3658 */ 3659 for (i = 0; i < 5; i++) { 3660 3661 if (is_kasan_insn(insn) || is_ubsan_insn(insn)) 3662 return true; 3663 3664 if (insn->type == INSN_JUMP_UNCONDITIONAL) { 3665 if (insn->jump_dest && 3666 insn->jump_dest->func == insn->func) { 3667 insn = insn->jump_dest; 3668 continue; 3669 } 3670 3671 break; 3672 } 3673 3674 if (insn->offset + insn->len >= insn->func->offset + insn->func->len) 3675 break; 3676 3677 insn = list_next_entry(insn, list); 3678 } 3679 3680 return false; 3681 } 3682 3683 static int validate_symbol(struct objtool_file *file, struct section *sec, 3684 struct symbol *sym, struct insn_state *state) 3685 { 3686 struct instruction *insn; 3687 int ret; 3688 3689 if (!sym->len) { 3690 WARN("%s() is missing an ELF size annotation", sym->name); 3691 return 1; 3692 } 3693 3694 if (sym->pfunc != sym || sym->alias != sym) 3695 return 0; 3696 3697 insn = find_insn(file, sec, sym->offset); 3698 if (!insn || insn->ignore || insn->visited) 3699 return 0; 3700 3701 state->uaccess = sym->uaccess_safe; 3702 3703 ret = validate_branch(file, insn->func, insn, *state); 3704 if (ret && backtrace) 3705 BT_FUNC("<=== (sym)", insn); 3706 return ret; 3707 } 3708 3709 static int validate_section(struct objtool_file *file, struct section *sec) 3710 { 3711 struct insn_state state; 3712 struct symbol *func; 3713 int warnings = 0; 3714 3715 list_for_each_entry(func, &sec->symbol_list, list) { 3716 if (func->type != STT_FUNC) 3717 continue; 3718 3719 init_insn_state(&state, sec); 3720 set_func_state(&state.cfi); 3721 3722 warnings += validate_symbol(file, sec, func, &state); 3723 } 3724 3725 return warnings; 3726 } 3727 3728 static int validate_vmlinux_functions(struct objtool_file *file) 3729 { 3730 struct section *sec; 3731 int warnings = 0; 3732 3733 sec = find_section_by_name(file->elf, ".noinstr.text"); 3734 if (sec) { 3735 warnings += validate_section(file, sec); 3736 warnings += validate_unwind_hints(file, sec); 3737 } 3738 3739 sec = find_section_by_name(file->elf, ".entry.text"); 3740 if (sec) { 3741 warnings += validate_section(file, sec); 3742 warnings += validate_unwind_hints(file, sec); 3743 } 3744 3745 return warnings; 3746 } 3747 3748 static int validate_functions(struct objtool_file *file) 3749 { 3750 struct section *sec; 3751 int warnings = 0; 3752 3753 for_each_sec(file, sec) { 3754 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 3755 continue; 3756 3757 warnings += validate_section(file, sec); 3758 } 3759 3760 return warnings; 3761 } 3762 3763 static int validate_ibt(struct objtool_file *file) 3764 { 3765 struct section *sec; 3766 struct reloc *reloc; 3767 3768 for_each_sec(file, sec) { 3769 bool is_data; 3770 3771 /* already done in validate_branch() */ 3772 if (sec->sh.sh_flags & SHF_EXECINSTR) 3773 continue; 3774 3775 if (!sec->reloc) 3776 continue; 3777 3778 if (!strncmp(sec->name, ".orc", 4)) 3779 continue; 3780 3781 if (!strncmp(sec->name, ".discard", 8)) 3782 continue; 3783 3784 if (!strncmp(sec->name, ".debug", 6)) 3785 continue; 3786 3787 if (!strcmp(sec->name, "_error_injection_whitelist")) 3788 continue; 3789 3790 if (!strcmp(sec->name, "_kprobe_blacklist")) 3791 continue; 3792 3793 is_data = strstr(sec->name, ".data") || strstr(sec->name, ".rodata"); 3794 3795 list_for_each_entry(reloc, &sec->reloc->reloc_list, list) { 3796 struct instruction *dest; 3797 3798 dest = validate_ibt_reloc(file, reloc); 3799 if (is_data && dest && !dest->noendbr) { 3800 warn_noendbr("data ", reloc->sym->sec, 3801 reloc->sym->offset + reloc->addend, 3802 dest); 3803 } 3804 } 3805 } 3806 3807 return 0; 3808 } 3809 3810 static int validate_reachable_instructions(struct objtool_file *file) 3811 { 3812 struct instruction *insn; 3813 3814 if (file->ignore_unreachables) 3815 return 0; 3816 3817 for_each_insn(file, insn) { 3818 if (insn->visited || ignore_unreachable_insn(file, insn)) 3819 continue; 3820 3821 WARN_FUNC("unreachable instruction", insn->sec, insn->offset); 3822 return 1; 3823 } 3824 3825 return 0; 3826 } 3827 3828 int check(struct objtool_file *file) 3829 { 3830 int ret, warnings = 0; 3831 3832 if (lto && !(vmlinux || module)) { 3833 fprintf(stderr, "--lto requires: --vmlinux or --module\n"); 3834 return 1; 3835 } 3836 3837 if (ibt && !lto) { 3838 fprintf(stderr, "--ibt requires: --lto\n"); 3839 return 1; 3840 } 3841 3842 arch_initial_func_cfi_state(&initial_func_cfi); 3843 init_cfi_state(&init_cfi); 3844 init_cfi_state(&func_cfi); 3845 set_func_state(&func_cfi); 3846 3847 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) 3848 goto out; 3849 3850 cfi_hash_add(&init_cfi); 3851 cfi_hash_add(&func_cfi); 3852 3853 ret = decode_sections(file); 3854 if (ret < 0) 3855 goto out; 3856 3857 warnings += ret; 3858 3859 if (list_empty(&file->insn_list)) 3860 goto out; 3861 3862 if (vmlinux && !lto) { 3863 ret = validate_vmlinux_functions(file); 3864 if (ret < 0) 3865 goto out; 3866 3867 warnings += ret; 3868 goto out; 3869 } 3870 3871 if (retpoline) { 3872 ret = validate_retpoline(file); 3873 if (ret < 0) 3874 return ret; 3875 warnings += ret; 3876 } 3877 3878 ret = validate_functions(file); 3879 if (ret < 0) 3880 goto out; 3881 warnings += ret; 3882 3883 ret = validate_unwind_hints(file, NULL); 3884 if (ret < 0) 3885 goto out; 3886 warnings += ret; 3887 3888 if (ibt) { 3889 ret = validate_ibt(file); 3890 if (ret < 0) 3891 goto out; 3892 warnings += ret; 3893 } 3894 3895 if (!warnings) { 3896 ret = validate_reachable_instructions(file); 3897 if (ret < 0) 3898 goto out; 3899 warnings += ret; 3900 } 3901 3902 ret = create_static_call_sections(file); 3903 if (ret < 0) 3904 goto out; 3905 warnings += ret; 3906 3907 if (retpoline) { 3908 ret = create_retpoline_sites_sections(file); 3909 if (ret < 0) 3910 goto out; 3911 warnings += ret; 3912 } 3913 3914 if (mcount) { 3915 ret = create_mcount_loc_sections(file); 3916 if (ret < 0) 3917 goto out; 3918 warnings += ret; 3919 } 3920 3921 if (ibt) { 3922 ret = create_ibt_endbr_seal_sections(file); 3923 if (ret < 0) 3924 goto out; 3925 warnings += ret; 3926 } 3927 3928 if (stats) { 3929 printf("nr_insns_visited: %ld\n", nr_insns_visited); 3930 printf("nr_cfi: %ld\n", nr_cfi); 3931 printf("nr_cfi_reused: %ld\n", nr_cfi_reused); 3932 printf("nr_cfi_cache: %ld\n", nr_cfi_cache); 3933 } 3934 3935 out: 3936 /* 3937 * For now, don't fail the kernel build on fatal warnings. These 3938 * errors are still fairly common due to the growing matrix of 3939 * supported toolchains and their recent pace of change. 3940 */ 3941 return 0; 3942 } 3943