1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6 #include <string.h> 7 #include <stdlib.h> 8 #include <inttypes.h> 9 #include <sys/mman.h> 10 11 #include <objtool/builtin.h> 12 #include <objtool/cfi.h> 13 #include <objtool/arch.h> 14 #include <objtool/check.h> 15 #include <objtool/special.h> 16 #include <objtool/warn.h> 17 #include <objtool/endianness.h> 18 19 #include <linux/objtool_types.h> 20 #include <linux/hashtable.h> 21 #include <linux/kernel.h> 22 #include <linux/static_call_types.h> 23 #include <linux/string.h> 24 25 struct alternative { 26 struct alternative *next; 27 struct instruction *insn; 28 bool skip_orig; 29 }; 30 31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 32 33 static struct cfi_init_state initial_func_cfi; 34 static struct cfi_state init_cfi; 35 static struct cfi_state func_cfi; 36 static struct cfi_state force_undefined_cfi; 37 38 struct instruction *find_insn(struct objtool_file *file, 39 struct section *sec, unsigned long offset) 40 { 41 struct instruction *insn; 42 43 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) { 44 if (insn->sec == sec && insn->offset == offset) 45 return insn; 46 } 47 48 return NULL; 49 } 50 51 struct instruction *next_insn_same_sec(struct objtool_file *file, 52 struct instruction *insn) 53 { 54 if (insn->idx == INSN_CHUNK_MAX) 55 return find_insn(file, insn->sec, insn->offset + insn->len); 56 57 insn++; 58 if (!insn->len) 59 return NULL; 60 61 return insn; 62 } 63 64 static struct instruction *next_insn_same_func(struct objtool_file *file, 65 struct instruction *insn) 66 { 67 struct instruction *next = next_insn_same_sec(file, insn); 68 struct symbol *func = insn_func(insn); 69 70 if (!func) 71 return NULL; 72 73 if (next && insn_func(next) == func) 74 return next; 75 76 /* Check if we're already in the subfunction: */ 77 if (func == func->cfunc) 78 return NULL; 79 80 /* Move to the subfunction: */ 81 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 82 } 83 84 static struct instruction *prev_insn_same_sec(struct objtool_file *file, 85 struct instruction *insn) 86 { 87 if (insn->idx == 0) { 88 if (insn->prev_len) 89 return find_insn(file, insn->sec, insn->offset - insn->prev_len); 90 return NULL; 91 } 92 93 return insn - 1; 94 } 95 96 static struct instruction *prev_insn_same_sym(struct objtool_file *file, 97 struct instruction *insn) 98 { 99 struct instruction *prev = prev_insn_same_sec(file, insn); 100 101 if (prev && insn_func(prev) == insn_func(insn)) 102 return prev; 103 104 return NULL; 105 } 106 107 #define for_each_insn(file, insn) \ 108 for (struct section *__sec, *__fake = (struct section *)1; \ 109 __fake; __fake = NULL) \ 110 for_each_sec(file, __sec) \ 111 sec_for_each_insn(file, __sec, insn) 112 113 #define func_for_each_insn(file, func, insn) \ 114 for (insn = find_insn(file, func->sec, func->offset); \ 115 insn; \ 116 insn = next_insn_same_func(file, insn)) 117 118 #define sym_for_each_insn(file, sym, insn) \ 119 for (insn = find_insn(file, sym->sec, sym->offset); \ 120 insn && insn->offset < sym->offset + sym->len; \ 121 insn = next_insn_same_sec(file, insn)) 122 123 #define sym_for_each_insn_continue_reverse(file, sym, insn) \ 124 for (insn = prev_insn_same_sec(file, insn); \ 125 insn && insn->offset >= sym->offset; \ 126 insn = prev_insn_same_sec(file, insn)) 127 128 #define sec_for_each_insn_from(file, insn) \ 129 for (; insn; insn = next_insn_same_sec(file, insn)) 130 131 #define sec_for_each_insn_continue(file, insn) \ 132 for (insn = next_insn_same_sec(file, insn); insn; \ 133 insn = next_insn_same_sec(file, insn)) 134 135 static inline struct symbol *insn_call_dest(struct instruction *insn) 136 { 137 if (insn->type == INSN_JUMP_DYNAMIC || 138 insn->type == INSN_CALL_DYNAMIC) 139 return NULL; 140 141 return insn->_call_dest; 142 } 143 144 static inline struct reloc *insn_jump_table(struct instruction *insn) 145 { 146 if (insn->type == INSN_JUMP_DYNAMIC || 147 insn->type == INSN_CALL_DYNAMIC) 148 return insn->_jump_table; 149 150 return NULL; 151 } 152 153 static bool is_jump_table_jump(struct instruction *insn) 154 { 155 struct alt_group *alt_group = insn->alt_group; 156 157 if (insn_jump_table(insn)) 158 return true; 159 160 /* Retpoline alternative for a jump table? */ 161 return alt_group && alt_group->orig_group && 162 insn_jump_table(alt_group->orig_group->first_insn); 163 } 164 165 static bool is_sibling_call(struct instruction *insn) 166 { 167 /* 168 * Assume only STT_FUNC calls have jump-tables. 169 */ 170 if (insn_func(insn)) { 171 /* An indirect jump is either a sibling call or a jump to a table. */ 172 if (insn->type == INSN_JUMP_DYNAMIC) 173 return !is_jump_table_jump(insn); 174 } 175 176 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */ 177 return (is_static_jump(insn) && insn_call_dest(insn)); 178 } 179 180 /* 181 * Checks if a string ends with another. 182 */ 183 static bool str_ends_with(const char *s, const char *sub) 184 { 185 const int slen = strlen(s); 186 const int sublen = strlen(sub); 187 188 if (sublen > slen) 189 return 0; 190 191 return !memcmp(s + slen - sublen, sub, sublen); 192 } 193 194 /* 195 * Checks if a function is a Rust "noreturn" one. 196 */ 197 static bool is_rust_noreturn(const struct symbol *func) 198 { 199 /* 200 * If it does not start with "_R", then it is not a Rust symbol. 201 */ 202 if (strncmp(func->name, "_R", 2)) 203 return false; 204 205 /* 206 * These are just heuristics -- we do not control the precise symbol 207 * name, due to the crate disambiguators (which depend on the compiler) 208 * as well as changes to the source code itself between versions (since 209 * these come from the Rust standard library). 210 */ 211 return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || 212 str_ends_with(func->name, "_4core6option13unwrap_failed") || 213 str_ends_with(func->name, "_4core6result13unwrap_failed") || 214 str_ends_with(func->name, "_4core9panicking5panic") || 215 str_ends_with(func->name, "_4core9panicking9panic_fmt") || 216 str_ends_with(func->name, "_4core9panicking14panic_explicit") || 217 str_ends_with(func->name, "_4core9panicking14panic_nounwind") || 218 str_ends_with(func->name, "_4core9panicking18panic_bounds_check") || 219 str_ends_with(func->name, "_4core9panicking19assert_failed_inner") || 220 str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") || 221 strstr(func->name, "_4core9panicking11panic_const24panic_const_") || 222 (strstr(func->name, "_4core5slice5index24slice_") && 223 str_ends_with(func->name, "_fail")); 224 } 225 226 /* 227 * This checks to see if the given function is a "noreturn" function. 228 * 229 * For global functions which are outside the scope of this object file, we 230 * have to keep a manual list of them. 231 * 232 * For local functions, we have to detect them manually by simply looking for 233 * the lack of a return instruction. 234 */ 235 static bool __dead_end_function(struct objtool_file *file, struct symbol *func, 236 int recursion) 237 { 238 int i; 239 struct instruction *insn; 240 bool empty = true; 241 242 #define NORETURN(func) __stringify(func), 243 static const char * const global_noreturns[] = { 244 #include "noreturns.h" 245 }; 246 #undef NORETURN 247 248 if (!func) 249 return false; 250 251 if (func->bind == STB_GLOBAL || func->bind == STB_WEAK) { 252 if (is_rust_noreturn(func)) 253 return true; 254 255 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) 256 if (!strcmp(func->name, global_noreturns[i])) 257 return true; 258 } 259 260 if (func->bind == STB_WEAK) 261 return false; 262 263 if (!func->len) 264 return false; 265 266 insn = find_insn(file, func->sec, func->offset); 267 if (!insn || !insn_func(insn)) 268 return false; 269 270 func_for_each_insn(file, func, insn) { 271 empty = false; 272 273 if (insn->type == INSN_RETURN) 274 return false; 275 } 276 277 if (empty) 278 return false; 279 280 /* 281 * A function can have a sibling call instead of a return. In that 282 * case, the function's dead-end status depends on whether the target 283 * of the sibling call returns. 284 */ 285 func_for_each_insn(file, func, insn) { 286 if (is_sibling_call(insn)) { 287 struct instruction *dest = insn->jump_dest; 288 289 if (!dest) 290 /* sibling call to another file */ 291 return false; 292 293 /* local sibling call */ 294 if (recursion == 5) { 295 /* 296 * Infinite recursion: two functions have 297 * sibling calls to each other. This is a very 298 * rare case. It means they aren't dead ends. 299 */ 300 return false; 301 } 302 303 return __dead_end_function(file, insn_func(dest), recursion+1); 304 } 305 } 306 307 return true; 308 } 309 310 static bool dead_end_function(struct objtool_file *file, struct symbol *func) 311 { 312 return __dead_end_function(file, func, 0); 313 } 314 315 static void init_cfi_state(struct cfi_state *cfi) 316 { 317 int i; 318 319 for (i = 0; i < CFI_NUM_REGS; i++) { 320 cfi->regs[i].base = CFI_UNDEFINED; 321 cfi->vals[i].base = CFI_UNDEFINED; 322 } 323 cfi->cfa.base = CFI_UNDEFINED; 324 cfi->drap_reg = CFI_UNDEFINED; 325 cfi->drap_offset = -1; 326 } 327 328 static void init_insn_state(struct objtool_file *file, struct insn_state *state, 329 struct section *sec) 330 { 331 memset(state, 0, sizeof(*state)); 332 init_cfi_state(&state->cfi); 333 334 /* 335 * We need the full vmlinux for noinstr validation, otherwise we can 336 * not correctly determine insn_call_dest(insn)->sec (external symbols 337 * do not have a section). 338 */ 339 if (opts.link && opts.noinstr && sec) 340 state->noinstr = sec->noinstr; 341 } 342 343 static struct cfi_state *cfi_alloc(void) 344 { 345 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state)); 346 if (!cfi) { 347 WARN("calloc failed"); 348 exit(1); 349 } 350 nr_cfi++; 351 return cfi; 352 } 353 354 static int cfi_bits; 355 static struct hlist_head *cfi_hash; 356 357 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2) 358 { 359 return memcmp((void *)cfi1 + sizeof(cfi1->hash), 360 (void *)cfi2 + sizeof(cfi2->hash), 361 sizeof(struct cfi_state) - sizeof(struct hlist_node)); 362 } 363 364 static inline u32 cfi_key(struct cfi_state *cfi) 365 { 366 return jhash((void *)cfi + sizeof(cfi->hash), 367 sizeof(*cfi) - sizeof(cfi->hash), 0); 368 } 369 370 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi) 371 { 372 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 373 struct cfi_state *obj; 374 375 hlist_for_each_entry(obj, head, hash) { 376 if (!cficmp(cfi, obj)) { 377 nr_cfi_cache++; 378 return obj; 379 } 380 } 381 382 obj = cfi_alloc(); 383 *obj = *cfi; 384 hlist_add_head(&obj->hash, head); 385 386 return obj; 387 } 388 389 static void cfi_hash_add(struct cfi_state *cfi) 390 { 391 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 392 393 hlist_add_head(&cfi->hash, head); 394 } 395 396 static void *cfi_hash_alloc(unsigned long size) 397 { 398 cfi_bits = max(10, ilog2(size)); 399 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits, 400 PROT_READ|PROT_WRITE, 401 MAP_PRIVATE|MAP_ANON, -1, 0); 402 if (cfi_hash == (void *)-1L) { 403 WARN("mmap fail cfi_hash"); 404 cfi_hash = NULL; 405 } else if (opts.stats) { 406 printf("cfi_bits: %d\n", cfi_bits); 407 } 408 409 return cfi_hash; 410 } 411 412 static unsigned long nr_insns; 413 static unsigned long nr_insns_visited; 414 415 /* 416 * Call the arch-specific instruction decoder for all the instructions and add 417 * them to the global instruction list. 418 */ 419 static int decode_instructions(struct objtool_file *file) 420 { 421 struct section *sec; 422 struct symbol *func; 423 unsigned long offset; 424 struct instruction *insn; 425 int ret; 426 427 for_each_sec(file, sec) { 428 struct instruction *insns = NULL; 429 u8 prev_len = 0; 430 u8 idx = 0; 431 432 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 433 continue; 434 435 if (strcmp(sec->name, ".altinstr_replacement") && 436 strcmp(sec->name, ".altinstr_aux") && 437 strncmp(sec->name, ".discard.", 9)) 438 sec->text = true; 439 440 if (!strcmp(sec->name, ".noinstr.text") || 441 !strcmp(sec->name, ".entry.text") || 442 !strcmp(sec->name, ".cpuidle.text") || 443 !strncmp(sec->name, ".text..__x86.", 13)) 444 sec->noinstr = true; 445 446 /* 447 * .init.text code is ran before userspace and thus doesn't 448 * strictly need retpolines, except for modules which are 449 * loaded late, they very much do need retpoline in their 450 * .init.text 451 */ 452 if (!strcmp(sec->name, ".init.text") && !opts.module) 453 sec->init = true; 454 455 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) { 456 if (!insns || idx == INSN_CHUNK_MAX) { 457 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE); 458 if (!insns) { 459 WARN("malloc failed"); 460 return -1; 461 } 462 idx = 0; 463 } else { 464 idx++; 465 } 466 insn = &insns[idx]; 467 insn->idx = idx; 468 469 INIT_LIST_HEAD(&insn->call_node); 470 insn->sec = sec; 471 insn->offset = offset; 472 insn->prev_len = prev_len; 473 474 ret = arch_decode_instruction(file, sec, offset, 475 sec->sh.sh_size - offset, 476 insn); 477 if (ret) 478 return ret; 479 480 prev_len = insn->len; 481 482 /* 483 * By default, "ud2" is a dead end unless otherwise 484 * annotated, because GCC 7 inserts it for certain 485 * divide-by-zero cases. 486 */ 487 if (insn->type == INSN_BUG) 488 insn->dead_end = true; 489 490 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset)); 491 nr_insns++; 492 } 493 494 // printf("%s: last chunk used: %d\n", sec->name, (int)idx); 495 496 sec_for_each_sym(sec, func) { 497 if (func->type != STT_NOTYPE && func->type != STT_FUNC) 498 continue; 499 500 if (func->offset == sec->sh.sh_size) { 501 /* Heuristic: likely an "end" symbol */ 502 if (func->type == STT_NOTYPE) 503 continue; 504 WARN("%s(): STT_FUNC at end of section", 505 func->name); 506 return -1; 507 } 508 509 if (func->embedded_insn || func->alias != func) 510 continue; 511 512 if (!find_insn(file, sec, func->offset)) { 513 WARN("%s(): can't find starting instruction", 514 func->name); 515 return -1; 516 } 517 518 sym_for_each_insn(file, func, insn) { 519 insn->sym = func; 520 if (func->type == STT_FUNC && 521 insn->type == INSN_ENDBR && 522 list_empty(&insn->call_node)) { 523 if (insn->offset == func->offset) { 524 list_add_tail(&insn->call_node, &file->endbr_list); 525 file->nr_endbr++; 526 } else { 527 file->nr_endbr_int++; 528 } 529 } 530 } 531 } 532 } 533 534 if (opts.stats) 535 printf("nr_insns: %lu\n", nr_insns); 536 537 return 0; 538 } 539 540 /* 541 * Read the pv_ops[] .data table to find the static initialized values. 542 */ 543 static int add_pv_ops(struct objtool_file *file, const char *symname) 544 { 545 struct symbol *sym, *func; 546 unsigned long off, end; 547 struct reloc *reloc; 548 int idx; 549 550 sym = find_symbol_by_name(file->elf, symname); 551 if (!sym) 552 return 0; 553 554 off = sym->offset; 555 end = off + sym->len; 556 for (;;) { 557 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 558 if (!reloc) 559 break; 560 561 func = reloc->sym; 562 if (func->type == STT_SECTION) 563 func = find_symbol_by_offset(reloc->sym->sec, 564 reloc_addend(reloc)); 565 566 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long); 567 568 objtool_pv_add(file, idx, func); 569 570 off = reloc_offset(reloc) + 1; 571 if (off > end) 572 break; 573 } 574 575 return 0; 576 } 577 578 /* 579 * Allocate and initialize file->pv_ops[]. 580 */ 581 static int init_pv_ops(struct objtool_file *file) 582 { 583 static const char *pv_ops_tables[] = { 584 "pv_ops", 585 "xen_cpu_ops", 586 "xen_irq_ops", 587 "xen_mmu_ops", 588 NULL, 589 }; 590 const char *pv_ops; 591 struct symbol *sym; 592 int idx, nr; 593 594 if (!opts.noinstr) 595 return 0; 596 597 file->pv_ops = NULL; 598 599 sym = find_symbol_by_name(file->elf, "pv_ops"); 600 if (!sym) 601 return 0; 602 603 nr = sym->len / sizeof(unsigned long); 604 file->pv_ops = calloc(sizeof(struct pv_state), nr); 605 if (!file->pv_ops) 606 return -1; 607 608 for (idx = 0; idx < nr; idx++) 609 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 610 611 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) 612 add_pv_ops(file, pv_ops); 613 614 return 0; 615 } 616 617 static struct instruction *find_last_insn(struct objtool_file *file, 618 struct section *sec) 619 { 620 struct instruction *insn = NULL; 621 unsigned int offset; 622 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0; 623 624 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--) 625 insn = find_insn(file, sec, offset); 626 627 return insn; 628 } 629 630 /* 631 * Mark "ud2" instructions and manually annotated dead ends. 632 */ 633 static int add_dead_ends(struct objtool_file *file) 634 { 635 struct section *rsec; 636 struct reloc *reloc; 637 struct instruction *insn; 638 uint64_t offset; 639 640 /* 641 * Check for manually annotated dead ends. 642 */ 643 rsec = find_section_by_name(file->elf, ".rela.discard.unreachable"); 644 if (!rsec) 645 goto reachable; 646 647 for_each_reloc(rsec, reloc) { 648 if (reloc->sym->type == STT_SECTION) { 649 offset = reloc_addend(reloc); 650 } else if (reloc->sym->local_label) { 651 offset = reloc->sym->offset; 652 } else { 653 WARN("unexpected relocation symbol type in %s", rsec->name); 654 return -1; 655 } 656 657 insn = find_insn(file, reloc->sym->sec, offset); 658 if (insn) 659 insn = prev_insn_same_sec(file, insn); 660 else if (offset == reloc->sym->sec->sh.sh_size) { 661 insn = find_last_insn(file, reloc->sym->sec); 662 if (!insn) { 663 WARN("can't find unreachable insn at %s+0x%" PRIx64, 664 reloc->sym->sec->name, offset); 665 return -1; 666 } 667 } else { 668 WARN("can't find unreachable insn at %s+0x%" PRIx64, 669 reloc->sym->sec->name, offset); 670 return -1; 671 } 672 673 insn->dead_end = true; 674 } 675 676 reachable: 677 /* 678 * These manually annotated reachable checks are needed for GCC 4.4, 679 * where the Linux unreachable() macro isn't supported. In that case 680 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's 681 * not a dead end. 682 */ 683 rsec = find_section_by_name(file->elf, ".rela.discard.reachable"); 684 if (!rsec) 685 return 0; 686 687 for_each_reloc(rsec, reloc) { 688 if (reloc->sym->type == STT_SECTION) { 689 offset = reloc_addend(reloc); 690 } else if (reloc->sym->local_label) { 691 offset = reloc->sym->offset; 692 } else { 693 WARN("unexpected relocation symbol type in %s", rsec->name); 694 return -1; 695 } 696 697 insn = find_insn(file, reloc->sym->sec, offset); 698 if (insn) 699 insn = prev_insn_same_sec(file, insn); 700 else if (offset == reloc->sym->sec->sh.sh_size) { 701 insn = find_last_insn(file, reloc->sym->sec); 702 if (!insn) { 703 WARN("can't find reachable insn at %s+0x%" PRIx64, 704 reloc->sym->sec->name, offset); 705 return -1; 706 } 707 } else { 708 WARN("can't find reachable insn at %s+0x%" PRIx64, 709 reloc->sym->sec->name, offset); 710 return -1; 711 } 712 713 insn->dead_end = false; 714 } 715 716 return 0; 717 } 718 719 static int create_static_call_sections(struct objtool_file *file) 720 { 721 struct static_call_site *site; 722 struct section *sec; 723 struct instruction *insn; 724 struct symbol *key_sym; 725 char *key_name, *tmp; 726 int idx; 727 728 sec = find_section_by_name(file->elf, ".static_call_sites"); 729 if (sec) { 730 INIT_LIST_HEAD(&file->static_call_list); 731 WARN("file already has .static_call_sites section, skipping"); 732 return 0; 733 } 734 735 if (list_empty(&file->static_call_list)) 736 return 0; 737 738 idx = 0; 739 list_for_each_entry(insn, &file->static_call_list, call_node) 740 idx++; 741 742 sec = elf_create_section_pair(file->elf, ".static_call_sites", 743 sizeof(*site), idx, idx * 2); 744 if (!sec) 745 return -1; 746 747 /* Allow modules to modify the low bits of static_call_site::key */ 748 sec->sh.sh_flags |= SHF_WRITE; 749 750 idx = 0; 751 list_for_each_entry(insn, &file->static_call_list, call_node) { 752 753 /* populate reloc for 'addr' */ 754 if (!elf_init_reloc_text_sym(file->elf, sec, 755 idx * sizeof(*site), idx * 2, 756 insn->sec, insn->offset)) 757 return -1; 758 759 /* find key symbol */ 760 key_name = strdup(insn_call_dest(insn)->name); 761 if (!key_name) { 762 perror("strdup"); 763 return -1; 764 } 765 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR, 766 STATIC_CALL_TRAMP_PREFIX_LEN)) { 767 WARN("static_call: trampoline name malformed: %s", key_name); 768 free(key_name); 769 return -1; 770 } 771 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN; 772 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN); 773 774 key_sym = find_symbol_by_name(file->elf, tmp); 775 if (!key_sym) { 776 if (!opts.module) { 777 WARN("static_call: can't find static_call_key symbol: %s", tmp); 778 free(key_name); 779 return -1; 780 } 781 782 /* 783 * For modules(), the key might not be exported, which 784 * means the module can make static calls but isn't 785 * allowed to change them. 786 * 787 * In that case we temporarily set the key to be the 788 * trampoline address. This is fixed up in 789 * static_call_add_module(). 790 */ 791 key_sym = insn_call_dest(insn); 792 } 793 free(key_name); 794 795 /* populate reloc for 'key' */ 796 if (!elf_init_reloc_data_sym(file->elf, sec, 797 idx * sizeof(*site) + 4, 798 (idx * 2) + 1, key_sym, 799 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL)) 800 return -1; 801 802 idx++; 803 } 804 805 return 0; 806 } 807 808 static int create_retpoline_sites_sections(struct objtool_file *file) 809 { 810 struct instruction *insn; 811 struct section *sec; 812 int idx; 813 814 sec = find_section_by_name(file->elf, ".retpoline_sites"); 815 if (sec) { 816 WARN("file already has .retpoline_sites, skipping"); 817 return 0; 818 } 819 820 idx = 0; 821 list_for_each_entry(insn, &file->retpoline_call_list, call_node) 822 idx++; 823 824 if (!idx) 825 return 0; 826 827 sec = elf_create_section_pair(file->elf, ".retpoline_sites", 828 sizeof(int), idx, idx); 829 if (!sec) 830 return -1; 831 832 idx = 0; 833 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 834 835 if (!elf_init_reloc_text_sym(file->elf, sec, 836 idx * sizeof(int), idx, 837 insn->sec, insn->offset)) 838 return -1; 839 840 idx++; 841 } 842 843 return 0; 844 } 845 846 static int create_return_sites_sections(struct objtool_file *file) 847 { 848 struct instruction *insn; 849 struct section *sec; 850 int idx; 851 852 sec = find_section_by_name(file->elf, ".return_sites"); 853 if (sec) { 854 WARN("file already has .return_sites, skipping"); 855 return 0; 856 } 857 858 idx = 0; 859 list_for_each_entry(insn, &file->return_thunk_list, call_node) 860 idx++; 861 862 if (!idx) 863 return 0; 864 865 sec = elf_create_section_pair(file->elf, ".return_sites", 866 sizeof(int), idx, idx); 867 if (!sec) 868 return -1; 869 870 idx = 0; 871 list_for_each_entry(insn, &file->return_thunk_list, call_node) { 872 873 if (!elf_init_reloc_text_sym(file->elf, sec, 874 idx * sizeof(int), idx, 875 insn->sec, insn->offset)) 876 return -1; 877 878 idx++; 879 } 880 881 return 0; 882 } 883 884 static int create_ibt_endbr_seal_sections(struct objtool_file *file) 885 { 886 struct instruction *insn; 887 struct section *sec; 888 int idx; 889 890 sec = find_section_by_name(file->elf, ".ibt_endbr_seal"); 891 if (sec) { 892 WARN("file already has .ibt_endbr_seal, skipping"); 893 return 0; 894 } 895 896 idx = 0; 897 list_for_each_entry(insn, &file->endbr_list, call_node) 898 idx++; 899 900 if (opts.stats) { 901 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); 902 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); 903 printf("ibt: superfluous ENDBR: %d\n", idx); 904 } 905 906 if (!idx) 907 return 0; 908 909 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal", 910 sizeof(int), idx, idx); 911 if (!sec) 912 return -1; 913 914 idx = 0; 915 list_for_each_entry(insn, &file->endbr_list, call_node) { 916 917 int *site = (int *)sec->data->d_buf + idx; 918 struct symbol *sym = insn->sym; 919 *site = 0; 920 921 if (opts.module && sym && sym->type == STT_FUNC && 922 insn->offset == sym->offset && 923 (!strcmp(sym->name, "init_module") || 924 !strcmp(sym->name, "cleanup_module"))) 925 WARN("%s(): not an indirect call target", sym->name); 926 927 if (!elf_init_reloc_text_sym(file->elf, sec, 928 idx * sizeof(int), idx, 929 insn->sec, insn->offset)) 930 return -1; 931 932 idx++; 933 } 934 935 return 0; 936 } 937 938 static int create_cfi_sections(struct objtool_file *file) 939 { 940 struct section *sec; 941 struct symbol *sym; 942 int idx; 943 944 sec = find_section_by_name(file->elf, ".cfi_sites"); 945 if (sec) { 946 INIT_LIST_HEAD(&file->call_list); 947 WARN("file already has .cfi_sites section, skipping"); 948 return 0; 949 } 950 951 idx = 0; 952 for_each_sym(file, sym) { 953 if (sym->type != STT_FUNC) 954 continue; 955 956 if (strncmp(sym->name, "__cfi_", 6)) 957 continue; 958 959 idx++; 960 } 961 962 sec = elf_create_section_pair(file->elf, ".cfi_sites", 963 sizeof(unsigned int), idx, idx); 964 if (!sec) 965 return -1; 966 967 idx = 0; 968 for_each_sym(file, sym) { 969 if (sym->type != STT_FUNC) 970 continue; 971 972 if (strncmp(sym->name, "__cfi_", 6)) 973 continue; 974 975 if (!elf_init_reloc_text_sym(file->elf, sec, 976 idx * sizeof(unsigned int), idx, 977 sym->sec, sym->offset)) 978 return -1; 979 980 idx++; 981 } 982 983 return 0; 984 } 985 986 static int create_mcount_loc_sections(struct objtool_file *file) 987 { 988 size_t addr_size = elf_addr_size(file->elf); 989 struct instruction *insn; 990 struct section *sec; 991 int idx; 992 993 sec = find_section_by_name(file->elf, "__mcount_loc"); 994 if (sec) { 995 INIT_LIST_HEAD(&file->mcount_loc_list); 996 WARN("file already has __mcount_loc section, skipping"); 997 return 0; 998 } 999 1000 if (list_empty(&file->mcount_loc_list)) 1001 return 0; 1002 1003 idx = 0; 1004 list_for_each_entry(insn, &file->mcount_loc_list, call_node) 1005 idx++; 1006 1007 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size, 1008 idx, idx); 1009 if (!sec) 1010 return -1; 1011 1012 sec->sh.sh_addralign = addr_size; 1013 1014 idx = 0; 1015 list_for_each_entry(insn, &file->mcount_loc_list, call_node) { 1016 1017 struct reloc *reloc; 1018 1019 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx, 1020 insn->sec, insn->offset); 1021 if (!reloc) 1022 return -1; 1023 1024 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32); 1025 1026 idx++; 1027 } 1028 1029 return 0; 1030 } 1031 1032 static int create_direct_call_sections(struct objtool_file *file) 1033 { 1034 struct instruction *insn; 1035 struct section *sec; 1036 int idx; 1037 1038 sec = find_section_by_name(file->elf, ".call_sites"); 1039 if (sec) { 1040 INIT_LIST_HEAD(&file->call_list); 1041 WARN("file already has .call_sites section, skipping"); 1042 return 0; 1043 } 1044 1045 if (list_empty(&file->call_list)) 1046 return 0; 1047 1048 idx = 0; 1049 list_for_each_entry(insn, &file->call_list, call_node) 1050 idx++; 1051 1052 sec = elf_create_section_pair(file->elf, ".call_sites", 1053 sizeof(unsigned int), idx, idx); 1054 if (!sec) 1055 return -1; 1056 1057 idx = 0; 1058 list_for_each_entry(insn, &file->call_list, call_node) { 1059 1060 if (!elf_init_reloc_text_sym(file->elf, sec, 1061 idx * sizeof(unsigned int), idx, 1062 insn->sec, insn->offset)) 1063 return -1; 1064 1065 idx++; 1066 } 1067 1068 return 0; 1069 } 1070 1071 /* 1072 * Warnings shouldn't be reported for ignored functions. 1073 */ 1074 static void add_ignores(struct objtool_file *file) 1075 { 1076 struct instruction *insn; 1077 struct section *rsec; 1078 struct symbol *func; 1079 struct reloc *reloc; 1080 1081 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard"); 1082 if (!rsec) 1083 return; 1084 1085 for_each_reloc(rsec, reloc) { 1086 switch (reloc->sym->type) { 1087 case STT_FUNC: 1088 func = reloc->sym; 1089 break; 1090 1091 case STT_SECTION: 1092 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc)); 1093 if (!func) 1094 continue; 1095 break; 1096 1097 default: 1098 WARN("unexpected relocation symbol type in %s: %d", 1099 rsec->name, reloc->sym->type); 1100 continue; 1101 } 1102 1103 func_for_each_insn(file, func, insn) 1104 insn->ignore = true; 1105 } 1106 } 1107 1108 /* 1109 * This is a whitelist of functions that is allowed to be called with AC set. 1110 * The list is meant to be minimal and only contains compiler instrumentation 1111 * ABI and a few functions used to implement *_{to,from}_user() functions. 1112 * 1113 * These functions must not directly change AC, but may PUSHF/POPF. 1114 */ 1115 static const char *uaccess_safe_builtin[] = { 1116 /* KASAN */ 1117 "kasan_report", 1118 "kasan_check_range", 1119 /* KASAN out-of-line */ 1120 "__asan_loadN_noabort", 1121 "__asan_load1_noabort", 1122 "__asan_load2_noabort", 1123 "__asan_load4_noabort", 1124 "__asan_load8_noabort", 1125 "__asan_load16_noabort", 1126 "__asan_storeN_noabort", 1127 "__asan_store1_noabort", 1128 "__asan_store2_noabort", 1129 "__asan_store4_noabort", 1130 "__asan_store8_noabort", 1131 "__asan_store16_noabort", 1132 "__kasan_check_read", 1133 "__kasan_check_write", 1134 /* KASAN in-line */ 1135 "__asan_report_load_n_noabort", 1136 "__asan_report_load1_noabort", 1137 "__asan_report_load2_noabort", 1138 "__asan_report_load4_noabort", 1139 "__asan_report_load8_noabort", 1140 "__asan_report_load16_noabort", 1141 "__asan_report_store_n_noabort", 1142 "__asan_report_store1_noabort", 1143 "__asan_report_store2_noabort", 1144 "__asan_report_store4_noabort", 1145 "__asan_report_store8_noabort", 1146 "__asan_report_store16_noabort", 1147 /* KCSAN */ 1148 "__kcsan_check_access", 1149 "__kcsan_mb", 1150 "__kcsan_wmb", 1151 "__kcsan_rmb", 1152 "__kcsan_release", 1153 "kcsan_found_watchpoint", 1154 "kcsan_setup_watchpoint", 1155 "kcsan_check_scoped_accesses", 1156 "kcsan_disable_current", 1157 "kcsan_enable_current_nowarn", 1158 /* KCSAN/TSAN */ 1159 "__tsan_func_entry", 1160 "__tsan_func_exit", 1161 "__tsan_read_range", 1162 "__tsan_write_range", 1163 "__tsan_read1", 1164 "__tsan_read2", 1165 "__tsan_read4", 1166 "__tsan_read8", 1167 "__tsan_read16", 1168 "__tsan_write1", 1169 "__tsan_write2", 1170 "__tsan_write4", 1171 "__tsan_write8", 1172 "__tsan_write16", 1173 "__tsan_read_write1", 1174 "__tsan_read_write2", 1175 "__tsan_read_write4", 1176 "__tsan_read_write8", 1177 "__tsan_read_write16", 1178 "__tsan_volatile_read1", 1179 "__tsan_volatile_read2", 1180 "__tsan_volatile_read4", 1181 "__tsan_volatile_read8", 1182 "__tsan_volatile_read16", 1183 "__tsan_volatile_write1", 1184 "__tsan_volatile_write2", 1185 "__tsan_volatile_write4", 1186 "__tsan_volatile_write8", 1187 "__tsan_volatile_write16", 1188 "__tsan_atomic8_load", 1189 "__tsan_atomic16_load", 1190 "__tsan_atomic32_load", 1191 "__tsan_atomic64_load", 1192 "__tsan_atomic8_store", 1193 "__tsan_atomic16_store", 1194 "__tsan_atomic32_store", 1195 "__tsan_atomic64_store", 1196 "__tsan_atomic8_exchange", 1197 "__tsan_atomic16_exchange", 1198 "__tsan_atomic32_exchange", 1199 "__tsan_atomic64_exchange", 1200 "__tsan_atomic8_fetch_add", 1201 "__tsan_atomic16_fetch_add", 1202 "__tsan_atomic32_fetch_add", 1203 "__tsan_atomic64_fetch_add", 1204 "__tsan_atomic8_fetch_sub", 1205 "__tsan_atomic16_fetch_sub", 1206 "__tsan_atomic32_fetch_sub", 1207 "__tsan_atomic64_fetch_sub", 1208 "__tsan_atomic8_fetch_and", 1209 "__tsan_atomic16_fetch_and", 1210 "__tsan_atomic32_fetch_and", 1211 "__tsan_atomic64_fetch_and", 1212 "__tsan_atomic8_fetch_or", 1213 "__tsan_atomic16_fetch_or", 1214 "__tsan_atomic32_fetch_or", 1215 "__tsan_atomic64_fetch_or", 1216 "__tsan_atomic8_fetch_xor", 1217 "__tsan_atomic16_fetch_xor", 1218 "__tsan_atomic32_fetch_xor", 1219 "__tsan_atomic64_fetch_xor", 1220 "__tsan_atomic8_fetch_nand", 1221 "__tsan_atomic16_fetch_nand", 1222 "__tsan_atomic32_fetch_nand", 1223 "__tsan_atomic64_fetch_nand", 1224 "__tsan_atomic8_compare_exchange_strong", 1225 "__tsan_atomic16_compare_exchange_strong", 1226 "__tsan_atomic32_compare_exchange_strong", 1227 "__tsan_atomic64_compare_exchange_strong", 1228 "__tsan_atomic8_compare_exchange_weak", 1229 "__tsan_atomic16_compare_exchange_weak", 1230 "__tsan_atomic32_compare_exchange_weak", 1231 "__tsan_atomic64_compare_exchange_weak", 1232 "__tsan_atomic8_compare_exchange_val", 1233 "__tsan_atomic16_compare_exchange_val", 1234 "__tsan_atomic32_compare_exchange_val", 1235 "__tsan_atomic64_compare_exchange_val", 1236 "__tsan_atomic_thread_fence", 1237 "__tsan_atomic_signal_fence", 1238 "__tsan_unaligned_read16", 1239 "__tsan_unaligned_write16", 1240 /* KCOV */ 1241 "write_comp_data", 1242 "check_kcov_mode", 1243 "__sanitizer_cov_trace_pc", 1244 "__sanitizer_cov_trace_const_cmp1", 1245 "__sanitizer_cov_trace_const_cmp2", 1246 "__sanitizer_cov_trace_const_cmp4", 1247 "__sanitizer_cov_trace_const_cmp8", 1248 "__sanitizer_cov_trace_cmp1", 1249 "__sanitizer_cov_trace_cmp2", 1250 "__sanitizer_cov_trace_cmp4", 1251 "__sanitizer_cov_trace_cmp8", 1252 "__sanitizer_cov_trace_switch", 1253 /* KMSAN */ 1254 "kmsan_copy_to_user", 1255 "kmsan_disable_current", 1256 "kmsan_enable_current", 1257 "kmsan_report", 1258 "kmsan_unpoison_entry_regs", 1259 "kmsan_unpoison_memory", 1260 "__msan_chain_origin", 1261 "__msan_get_context_state", 1262 "__msan_instrument_asm_store", 1263 "__msan_metadata_ptr_for_load_1", 1264 "__msan_metadata_ptr_for_load_2", 1265 "__msan_metadata_ptr_for_load_4", 1266 "__msan_metadata_ptr_for_load_8", 1267 "__msan_metadata_ptr_for_load_n", 1268 "__msan_metadata_ptr_for_store_1", 1269 "__msan_metadata_ptr_for_store_2", 1270 "__msan_metadata_ptr_for_store_4", 1271 "__msan_metadata_ptr_for_store_8", 1272 "__msan_metadata_ptr_for_store_n", 1273 "__msan_poison_alloca", 1274 "__msan_warning", 1275 /* UBSAN */ 1276 "ubsan_type_mismatch_common", 1277 "__ubsan_handle_type_mismatch", 1278 "__ubsan_handle_type_mismatch_v1", 1279 "__ubsan_handle_shift_out_of_bounds", 1280 "__ubsan_handle_load_invalid_value", 1281 /* STACKLEAK */ 1282 "stackleak_track_stack", 1283 /* misc */ 1284 "csum_partial_copy_generic", 1285 "copy_mc_fragile", 1286 "copy_mc_fragile_handle_tail", 1287 "copy_mc_enhanced_fast_string", 1288 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */ 1289 "rep_stos_alternative", 1290 "rep_movs_alternative", 1291 "__copy_user_nocache", 1292 NULL 1293 }; 1294 1295 static void add_uaccess_safe(struct objtool_file *file) 1296 { 1297 struct symbol *func; 1298 const char **name; 1299 1300 if (!opts.uaccess) 1301 return; 1302 1303 for (name = uaccess_safe_builtin; *name; name++) { 1304 func = find_symbol_by_name(file->elf, *name); 1305 if (!func) 1306 continue; 1307 1308 func->uaccess_safe = true; 1309 } 1310 } 1311 1312 /* 1313 * FIXME: For now, just ignore any alternatives which add retpolines. This is 1314 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline. 1315 * But it at least allows objtool to understand the control flow *around* the 1316 * retpoline. 1317 */ 1318 static int add_ignore_alternatives(struct objtool_file *file) 1319 { 1320 struct section *rsec; 1321 struct reloc *reloc; 1322 struct instruction *insn; 1323 1324 rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts"); 1325 if (!rsec) 1326 return 0; 1327 1328 for_each_reloc(rsec, reloc) { 1329 if (reloc->sym->type != STT_SECTION) { 1330 WARN("unexpected relocation symbol type in %s", rsec->name); 1331 return -1; 1332 } 1333 1334 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); 1335 if (!insn) { 1336 WARN("bad .discard.ignore_alts entry"); 1337 return -1; 1338 } 1339 1340 insn->ignore_alts = true; 1341 } 1342 1343 return 0; 1344 } 1345 1346 /* 1347 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol 1348 * will be added to the .retpoline_sites section. 1349 */ 1350 __weak bool arch_is_retpoline(struct symbol *sym) 1351 { 1352 return false; 1353 } 1354 1355 /* 1356 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol 1357 * will be added to the .return_sites section. 1358 */ 1359 __weak bool arch_is_rethunk(struct symbol *sym) 1360 { 1361 return false; 1362 } 1363 1364 /* 1365 * Symbols that are embedded inside other instructions, because sometimes crazy 1366 * code exists. These are mostly ignored for validation purposes. 1367 */ 1368 __weak bool arch_is_embedded_insn(struct symbol *sym) 1369 { 1370 return false; 1371 } 1372 1373 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) 1374 { 1375 struct reloc *reloc; 1376 1377 if (insn->no_reloc) 1378 return NULL; 1379 1380 if (!file) 1381 return NULL; 1382 1383 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 1384 insn->offset, insn->len); 1385 if (!reloc) { 1386 insn->no_reloc = 1; 1387 return NULL; 1388 } 1389 1390 return reloc; 1391 } 1392 1393 static void remove_insn_ops(struct instruction *insn) 1394 { 1395 struct stack_op *op, *next; 1396 1397 for (op = insn->stack_ops; op; op = next) { 1398 next = op->next; 1399 free(op); 1400 } 1401 insn->stack_ops = NULL; 1402 } 1403 1404 static void annotate_call_site(struct objtool_file *file, 1405 struct instruction *insn, bool sibling) 1406 { 1407 struct reloc *reloc = insn_reloc(file, insn); 1408 struct symbol *sym = insn_call_dest(insn); 1409 1410 if (!sym) 1411 sym = reloc->sym; 1412 1413 /* 1414 * Alternative replacement code is just template code which is 1415 * sometimes copied to the original instruction. For now, don't 1416 * annotate it. (In the future we might consider annotating the 1417 * original instruction if/when it ever makes sense to do so.) 1418 */ 1419 if (!strcmp(insn->sec->name, ".altinstr_replacement")) 1420 return; 1421 1422 if (sym->static_call_tramp) { 1423 list_add_tail(&insn->call_node, &file->static_call_list); 1424 return; 1425 } 1426 1427 if (sym->retpoline_thunk) { 1428 list_add_tail(&insn->call_node, &file->retpoline_call_list); 1429 return; 1430 } 1431 1432 /* 1433 * Many compilers cannot disable KCOV or sanitizer calls with a function 1434 * attribute so they need a little help, NOP out any such calls from 1435 * noinstr text. 1436 */ 1437 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) { 1438 if (reloc) 1439 set_reloc_type(file->elf, reloc, R_NONE); 1440 1441 elf_write_insn(file->elf, insn->sec, 1442 insn->offset, insn->len, 1443 sibling ? arch_ret_insn(insn->len) 1444 : arch_nop_insn(insn->len)); 1445 1446 insn->type = sibling ? INSN_RETURN : INSN_NOP; 1447 1448 if (sibling) { 1449 /* 1450 * We've replaced the tail-call JMP insn by two new 1451 * insn: RET; INT3, except we only have a single struct 1452 * insn here. Mark it retpoline_safe to avoid the SLS 1453 * warning, instead of adding another insn. 1454 */ 1455 insn->retpoline_safe = true; 1456 } 1457 1458 return; 1459 } 1460 1461 if (opts.mcount && sym->fentry) { 1462 if (sibling) 1463 WARN_INSN(insn, "tail call to __fentry__ !?!?"); 1464 if (opts.mnop) { 1465 if (reloc) 1466 set_reloc_type(file->elf, reloc, R_NONE); 1467 1468 elf_write_insn(file->elf, insn->sec, 1469 insn->offset, insn->len, 1470 arch_nop_insn(insn->len)); 1471 1472 insn->type = INSN_NOP; 1473 } 1474 1475 list_add_tail(&insn->call_node, &file->mcount_loc_list); 1476 return; 1477 } 1478 1479 if (insn->type == INSN_CALL && !insn->sec->init) 1480 list_add_tail(&insn->call_node, &file->call_list); 1481 1482 if (!sibling && dead_end_function(file, sym)) 1483 insn->dead_end = true; 1484 } 1485 1486 static void add_call_dest(struct objtool_file *file, struct instruction *insn, 1487 struct symbol *dest, bool sibling) 1488 { 1489 insn->_call_dest = dest; 1490 if (!dest) 1491 return; 1492 1493 /* 1494 * Whatever stack impact regular CALLs have, should be undone 1495 * by the RETURN of the called function. 1496 * 1497 * Annotated intra-function calls retain the stack_ops but 1498 * are converted to JUMP, see read_intra_function_calls(). 1499 */ 1500 remove_insn_ops(insn); 1501 1502 annotate_call_site(file, insn, sibling); 1503 } 1504 1505 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn) 1506 { 1507 /* 1508 * Retpoline calls/jumps are really dynamic calls/jumps in disguise, 1509 * so convert them accordingly. 1510 */ 1511 switch (insn->type) { 1512 case INSN_CALL: 1513 insn->type = INSN_CALL_DYNAMIC; 1514 break; 1515 case INSN_JUMP_UNCONDITIONAL: 1516 insn->type = INSN_JUMP_DYNAMIC; 1517 break; 1518 case INSN_JUMP_CONDITIONAL: 1519 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL; 1520 break; 1521 default: 1522 return; 1523 } 1524 1525 insn->retpoline_safe = true; 1526 1527 /* 1528 * Whatever stack impact regular CALLs have, should be undone 1529 * by the RETURN of the called function. 1530 * 1531 * Annotated intra-function calls retain the stack_ops but 1532 * are converted to JUMP, see read_intra_function_calls(). 1533 */ 1534 remove_insn_ops(insn); 1535 1536 annotate_call_site(file, insn, false); 1537 } 1538 1539 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) 1540 { 1541 /* 1542 * Return thunk tail calls are really just returns in disguise, 1543 * so convert them accordingly. 1544 */ 1545 insn->type = INSN_RETURN; 1546 insn->retpoline_safe = true; 1547 1548 if (add) 1549 list_add_tail(&insn->call_node, &file->return_thunk_list); 1550 } 1551 1552 static bool is_first_func_insn(struct objtool_file *file, 1553 struct instruction *insn, struct symbol *sym) 1554 { 1555 if (insn->offset == sym->offset) 1556 return true; 1557 1558 /* Allow direct CALL/JMP past ENDBR */ 1559 if (opts.ibt) { 1560 struct instruction *prev = prev_insn_same_sym(file, insn); 1561 1562 if (prev && prev->type == INSN_ENDBR && 1563 insn->offset == sym->offset + prev->len) 1564 return true; 1565 } 1566 1567 return false; 1568 } 1569 1570 /* 1571 * A sibling call is a tail-call to another symbol -- to differentiate from a 1572 * recursive tail-call which is to the same symbol. 1573 */ 1574 static bool jump_is_sibling_call(struct objtool_file *file, 1575 struct instruction *from, struct instruction *to) 1576 { 1577 struct symbol *fs = from->sym; 1578 struct symbol *ts = to->sym; 1579 1580 /* Not a sibling call if from/to a symbol hole */ 1581 if (!fs || !ts) 1582 return false; 1583 1584 /* Not a sibling call if not targeting the start of a symbol. */ 1585 if (!is_first_func_insn(file, to, ts)) 1586 return false; 1587 1588 /* Disallow sibling calls into STT_NOTYPE */ 1589 if (ts->type == STT_NOTYPE) 1590 return false; 1591 1592 /* Must not be self to be a sibling */ 1593 return fs->pfunc != ts->pfunc; 1594 } 1595 1596 /* 1597 * Find the destination instructions for all jumps. 1598 */ 1599 static int add_jump_destinations(struct objtool_file *file) 1600 { 1601 struct instruction *insn, *jump_dest; 1602 struct reloc *reloc; 1603 struct section *dest_sec; 1604 unsigned long dest_off; 1605 1606 for_each_insn(file, insn) { 1607 if (insn->jump_dest) { 1608 /* 1609 * handle_group_alt() may have previously set 1610 * 'jump_dest' for some alternatives. 1611 */ 1612 continue; 1613 } 1614 if (!is_static_jump(insn)) 1615 continue; 1616 1617 reloc = insn_reloc(file, insn); 1618 if (!reloc) { 1619 dest_sec = insn->sec; 1620 dest_off = arch_jump_destination(insn); 1621 } else if (reloc->sym->type == STT_SECTION) { 1622 dest_sec = reloc->sym->sec; 1623 dest_off = arch_dest_reloc_offset(reloc_addend(reloc)); 1624 } else if (reloc->sym->retpoline_thunk) { 1625 add_retpoline_call(file, insn); 1626 continue; 1627 } else if (reloc->sym->return_thunk) { 1628 add_return_call(file, insn, true); 1629 continue; 1630 } else if (insn_func(insn)) { 1631 /* 1632 * External sibling call or internal sibling call with 1633 * STT_FUNC reloc. 1634 */ 1635 add_call_dest(file, insn, reloc->sym, true); 1636 continue; 1637 } else if (reloc->sym->sec->idx) { 1638 dest_sec = reloc->sym->sec; 1639 dest_off = reloc->sym->sym.st_value + 1640 arch_dest_reloc_offset(reloc_addend(reloc)); 1641 } else { 1642 /* non-func asm code jumping to another file */ 1643 continue; 1644 } 1645 1646 jump_dest = find_insn(file, dest_sec, dest_off); 1647 if (!jump_dest) { 1648 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); 1649 1650 /* 1651 * This is a special case for retbleed_untrain_ret(). 1652 * It jumps to __x86_return_thunk(), but objtool 1653 * can't find the thunk's starting RET 1654 * instruction, because the RET is also in the 1655 * middle of another instruction. Objtool only 1656 * knows about the outer instruction. 1657 */ 1658 if (sym && sym->embedded_insn) { 1659 add_return_call(file, insn, false); 1660 continue; 1661 } 1662 1663 WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx", 1664 dest_sec->name, dest_off); 1665 return -1; 1666 } 1667 1668 /* 1669 * An intra-TU jump in retpoline.o might not have a relocation 1670 * for its jump dest, in which case the above 1671 * add_{retpoline,return}_call() didn't happen. 1672 */ 1673 if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) { 1674 if (jump_dest->sym->retpoline_thunk) { 1675 add_retpoline_call(file, insn); 1676 continue; 1677 } 1678 if (jump_dest->sym->return_thunk) { 1679 add_return_call(file, insn, true); 1680 continue; 1681 } 1682 } 1683 1684 /* 1685 * Cross-function jump. 1686 */ 1687 if (insn_func(insn) && insn_func(jump_dest) && 1688 insn_func(insn) != insn_func(jump_dest)) { 1689 1690 /* 1691 * For GCC 8+, create parent/child links for any cold 1692 * subfunctions. This is _mostly_ redundant with a 1693 * similar initialization in read_symbols(). 1694 * 1695 * If a function has aliases, we want the *first* such 1696 * function in the symbol table to be the subfunction's 1697 * parent. In that case we overwrite the 1698 * initialization done in read_symbols(). 1699 * 1700 * However this code can't completely replace the 1701 * read_symbols() code because this doesn't detect the 1702 * case where the parent function's only reference to a 1703 * subfunction is through a jump table. 1704 */ 1705 if (!strstr(insn_func(insn)->name, ".cold") && 1706 strstr(insn_func(jump_dest)->name, ".cold")) { 1707 insn_func(insn)->cfunc = insn_func(jump_dest); 1708 insn_func(jump_dest)->pfunc = insn_func(insn); 1709 } 1710 } 1711 1712 if (jump_is_sibling_call(file, insn, jump_dest)) { 1713 /* 1714 * Internal sibling call without reloc or with 1715 * STT_SECTION reloc. 1716 */ 1717 add_call_dest(file, insn, insn_func(jump_dest), true); 1718 continue; 1719 } 1720 1721 insn->jump_dest = jump_dest; 1722 } 1723 1724 return 0; 1725 } 1726 1727 static struct symbol *find_call_destination(struct section *sec, unsigned long offset) 1728 { 1729 struct symbol *call_dest; 1730 1731 call_dest = find_func_by_offset(sec, offset); 1732 if (!call_dest) 1733 call_dest = find_symbol_by_offset(sec, offset); 1734 1735 return call_dest; 1736 } 1737 1738 /* 1739 * Find the destination instructions for all calls. 1740 */ 1741 static int add_call_destinations(struct objtool_file *file) 1742 { 1743 struct instruction *insn; 1744 unsigned long dest_off; 1745 struct symbol *dest; 1746 struct reloc *reloc; 1747 1748 for_each_insn(file, insn) { 1749 if (insn->type != INSN_CALL) 1750 continue; 1751 1752 reloc = insn_reloc(file, insn); 1753 if (!reloc) { 1754 dest_off = arch_jump_destination(insn); 1755 dest = find_call_destination(insn->sec, dest_off); 1756 1757 add_call_dest(file, insn, dest, false); 1758 1759 if (insn->ignore) 1760 continue; 1761 1762 if (!insn_call_dest(insn)) { 1763 WARN_INSN(insn, "unannotated intra-function call"); 1764 return -1; 1765 } 1766 1767 if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) { 1768 WARN_INSN(insn, "unsupported call to non-function"); 1769 return -1; 1770 } 1771 1772 } else if (reloc->sym->type == STT_SECTION) { 1773 dest_off = arch_dest_reloc_offset(reloc_addend(reloc)); 1774 dest = find_call_destination(reloc->sym->sec, dest_off); 1775 if (!dest) { 1776 WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx", 1777 reloc->sym->sec->name, dest_off); 1778 return -1; 1779 } 1780 1781 add_call_dest(file, insn, dest, false); 1782 1783 } else if (reloc->sym->retpoline_thunk) { 1784 add_retpoline_call(file, insn); 1785 1786 } else 1787 add_call_dest(file, insn, reloc->sym, false); 1788 } 1789 1790 return 0; 1791 } 1792 1793 /* 1794 * The .alternatives section requires some extra special care over and above 1795 * other special sections because alternatives are patched in place. 1796 */ 1797 static int handle_group_alt(struct objtool_file *file, 1798 struct special_alt *special_alt, 1799 struct instruction *orig_insn, 1800 struct instruction **new_insn) 1801 { 1802 struct instruction *last_new_insn = NULL, *insn, *nop = NULL; 1803 struct alt_group *orig_alt_group, *new_alt_group; 1804 unsigned long dest_off; 1805 1806 orig_alt_group = orig_insn->alt_group; 1807 if (!orig_alt_group) { 1808 struct instruction *last_orig_insn = NULL; 1809 1810 orig_alt_group = malloc(sizeof(*orig_alt_group)); 1811 if (!orig_alt_group) { 1812 WARN("malloc failed"); 1813 return -1; 1814 } 1815 orig_alt_group->cfi = calloc(special_alt->orig_len, 1816 sizeof(struct cfi_state *)); 1817 if (!orig_alt_group->cfi) { 1818 WARN("calloc failed"); 1819 return -1; 1820 } 1821 1822 insn = orig_insn; 1823 sec_for_each_insn_from(file, insn) { 1824 if (insn->offset >= special_alt->orig_off + special_alt->orig_len) 1825 break; 1826 1827 insn->alt_group = orig_alt_group; 1828 last_orig_insn = insn; 1829 } 1830 orig_alt_group->orig_group = NULL; 1831 orig_alt_group->first_insn = orig_insn; 1832 orig_alt_group->last_insn = last_orig_insn; 1833 orig_alt_group->nop = NULL; 1834 } else { 1835 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - 1836 orig_alt_group->first_insn->offset != special_alt->orig_len) { 1837 WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d", 1838 orig_alt_group->last_insn->offset + 1839 orig_alt_group->last_insn->len - 1840 orig_alt_group->first_insn->offset, 1841 special_alt->orig_len); 1842 return -1; 1843 } 1844 } 1845 1846 new_alt_group = malloc(sizeof(*new_alt_group)); 1847 if (!new_alt_group) { 1848 WARN("malloc failed"); 1849 return -1; 1850 } 1851 1852 if (special_alt->new_len < special_alt->orig_len) { 1853 /* 1854 * Insert a fake nop at the end to make the replacement 1855 * alt_group the same size as the original. This is needed to 1856 * allow propagate_alt_cfi() to do its magic. When the last 1857 * instruction affects the stack, the instruction after it (the 1858 * nop) will propagate the new state to the shared CFI array. 1859 */ 1860 nop = malloc(sizeof(*nop)); 1861 if (!nop) { 1862 WARN("malloc failed"); 1863 return -1; 1864 } 1865 memset(nop, 0, sizeof(*nop)); 1866 1867 nop->sec = special_alt->new_sec; 1868 nop->offset = special_alt->new_off + special_alt->new_len; 1869 nop->len = special_alt->orig_len - special_alt->new_len; 1870 nop->type = INSN_NOP; 1871 nop->sym = orig_insn->sym; 1872 nop->alt_group = new_alt_group; 1873 nop->ignore = orig_insn->ignore_alts; 1874 } 1875 1876 if (!special_alt->new_len) { 1877 *new_insn = nop; 1878 goto end; 1879 } 1880 1881 insn = *new_insn; 1882 sec_for_each_insn_from(file, insn) { 1883 struct reloc *alt_reloc; 1884 1885 if (insn->offset >= special_alt->new_off + special_alt->new_len) 1886 break; 1887 1888 last_new_insn = insn; 1889 1890 insn->ignore = orig_insn->ignore_alts; 1891 insn->sym = orig_insn->sym; 1892 insn->alt_group = new_alt_group; 1893 1894 /* 1895 * Since alternative replacement code is copy/pasted by the 1896 * kernel after applying relocations, generally such code can't 1897 * have relative-address relocation references to outside the 1898 * .altinstr_replacement section, unless the arch's 1899 * alternatives code can adjust the relative offsets 1900 * accordingly. 1901 */ 1902 alt_reloc = insn_reloc(file, insn); 1903 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) && 1904 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { 1905 1906 WARN_INSN(insn, "unsupported relocation in alternatives section"); 1907 return -1; 1908 } 1909 1910 if (!is_static_jump(insn)) 1911 continue; 1912 1913 if (!insn->immediate) 1914 continue; 1915 1916 dest_off = arch_jump_destination(insn); 1917 if (dest_off == special_alt->new_off + special_alt->new_len) { 1918 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn); 1919 if (!insn->jump_dest) { 1920 WARN_INSN(insn, "can't find alternative jump destination"); 1921 return -1; 1922 } 1923 } 1924 } 1925 1926 if (!last_new_insn) { 1927 WARN_FUNC("can't find last new alternative instruction", 1928 special_alt->new_sec, special_alt->new_off); 1929 return -1; 1930 } 1931 1932 end: 1933 new_alt_group->orig_group = orig_alt_group; 1934 new_alt_group->first_insn = *new_insn; 1935 new_alt_group->last_insn = last_new_insn; 1936 new_alt_group->nop = nop; 1937 new_alt_group->cfi = orig_alt_group->cfi; 1938 return 0; 1939 } 1940 1941 /* 1942 * A jump table entry can either convert a nop to a jump or a jump to a nop. 1943 * If the original instruction is a jump, make the alt entry an effective nop 1944 * by just skipping the original instruction. 1945 */ 1946 static int handle_jump_alt(struct objtool_file *file, 1947 struct special_alt *special_alt, 1948 struct instruction *orig_insn, 1949 struct instruction **new_insn) 1950 { 1951 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && 1952 orig_insn->type != INSN_NOP) { 1953 1954 WARN_INSN(orig_insn, "unsupported instruction at jump label"); 1955 return -1; 1956 } 1957 1958 if (opts.hack_jump_label && special_alt->key_addend & 2) { 1959 struct reloc *reloc = insn_reloc(file, orig_insn); 1960 1961 if (reloc) 1962 set_reloc_type(file->elf, reloc, R_NONE); 1963 elf_write_insn(file->elf, orig_insn->sec, 1964 orig_insn->offset, orig_insn->len, 1965 arch_nop_insn(orig_insn->len)); 1966 orig_insn->type = INSN_NOP; 1967 } 1968 1969 if (orig_insn->type == INSN_NOP) { 1970 if (orig_insn->len == 2) 1971 file->jl_nop_short++; 1972 else 1973 file->jl_nop_long++; 1974 1975 return 0; 1976 } 1977 1978 if (orig_insn->len == 2) 1979 file->jl_short++; 1980 else 1981 file->jl_long++; 1982 1983 *new_insn = next_insn_same_sec(file, orig_insn); 1984 return 0; 1985 } 1986 1987 /* 1988 * Read all the special sections which have alternate instructions which can be 1989 * patched in or redirected to at runtime. Each instruction having alternate 1990 * instruction(s) has them added to its insn->alts list, which will be 1991 * traversed in validate_branch(). 1992 */ 1993 static int add_special_section_alts(struct objtool_file *file) 1994 { 1995 struct list_head special_alts; 1996 struct instruction *orig_insn, *new_insn; 1997 struct special_alt *special_alt, *tmp; 1998 struct alternative *alt; 1999 int ret; 2000 2001 ret = special_get_alts(file->elf, &special_alts); 2002 if (ret) 2003 return ret; 2004 2005 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) { 2006 2007 orig_insn = find_insn(file, special_alt->orig_sec, 2008 special_alt->orig_off); 2009 if (!orig_insn) { 2010 WARN_FUNC("special: can't find orig instruction", 2011 special_alt->orig_sec, special_alt->orig_off); 2012 ret = -1; 2013 goto out; 2014 } 2015 2016 new_insn = NULL; 2017 if (!special_alt->group || special_alt->new_len) { 2018 new_insn = find_insn(file, special_alt->new_sec, 2019 special_alt->new_off); 2020 if (!new_insn) { 2021 WARN_FUNC("special: can't find new instruction", 2022 special_alt->new_sec, 2023 special_alt->new_off); 2024 ret = -1; 2025 goto out; 2026 } 2027 } 2028 2029 if (special_alt->group) { 2030 if (!special_alt->orig_len) { 2031 WARN_INSN(orig_insn, "empty alternative entry"); 2032 continue; 2033 } 2034 2035 ret = handle_group_alt(file, special_alt, orig_insn, 2036 &new_insn); 2037 if (ret) 2038 goto out; 2039 } else if (special_alt->jump_or_nop) { 2040 ret = handle_jump_alt(file, special_alt, orig_insn, 2041 &new_insn); 2042 if (ret) 2043 goto out; 2044 } 2045 2046 alt = malloc(sizeof(*alt)); 2047 if (!alt) { 2048 WARN("malloc failed"); 2049 ret = -1; 2050 goto out; 2051 } 2052 2053 alt->insn = new_insn; 2054 alt->skip_orig = special_alt->skip_orig; 2055 orig_insn->ignore_alts |= special_alt->skip_alt; 2056 alt->next = orig_insn->alts; 2057 orig_insn->alts = alt; 2058 2059 list_del(&special_alt->list); 2060 free(special_alt); 2061 } 2062 2063 if (opts.stats) { 2064 printf("jl\\\tNOP\tJMP\n"); 2065 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); 2066 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); 2067 } 2068 2069 out: 2070 return ret; 2071 } 2072 2073 static int add_jump_table(struct objtool_file *file, struct instruction *insn, 2074 struct reloc *next_table) 2075 { 2076 struct symbol *pfunc = insn_func(insn)->pfunc; 2077 struct reloc *table = insn_jump_table(insn); 2078 struct instruction *dest_insn; 2079 unsigned int prev_offset = 0; 2080 struct reloc *reloc = table; 2081 struct alternative *alt; 2082 2083 /* 2084 * Each @reloc is a switch table relocation which points to the target 2085 * instruction. 2086 */ 2087 for_each_reloc_from(table->sec, reloc) { 2088 2089 /* Check for the end of the table: */ 2090 if (reloc != table && reloc == next_table) 2091 break; 2092 2093 /* Make sure the table entries are consecutive: */ 2094 if (prev_offset && reloc_offset(reloc) != prev_offset + 8) 2095 break; 2096 2097 /* Detect function pointers from contiguous objects: */ 2098 if (reloc->sym->sec == pfunc->sec && 2099 reloc_addend(reloc) == pfunc->offset) 2100 break; 2101 2102 dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); 2103 if (!dest_insn) 2104 break; 2105 2106 /* Make sure the destination is in the same function: */ 2107 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc) 2108 break; 2109 2110 alt = malloc(sizeof(*alt)); 2111 if (!alt) { 2112 WARN("malloc failed"); 2113 return -1; 2114 } 2115 2116 alt->insn = dest_insn; 2117 alt->next = insn->alts; 2118 insn->alts = alt; 2119 prev_offset = reloc_offset(reloc); 2120 } 2121 2122 if (!prev_offset) { 2123 WARN_INSN(insn, "can't find switch jump table"); 2124 return -1; 2125 } 2126 2127 return 0; 2128 } 2129 2130 /* 2131 * find_jump_table() - Given a dynamic jump, find the switch jump table 2132 * associated with it. 2133 */ 2134 static struct reloc *find_jump_table(struct objtool_file *file, 2135 struct symbol *func, 2136 struct instruction *insn) 2137 { 2138 struct reloc *table_reloc; 2139 struct instruction *dest_insn, *orig_insn = insn; 2140 2141 /* 2142 * Backward search using the @first_jump_src links, these help avoid 2143 * much of the 'in between' code. Which avoids us getting confused by 2144 * it. 2145 */ 2146 for (; 2147 insn && insn_func(insn) && insn_func(insn)->pfunc == func; 2148 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) { 2149 2150 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC) 2151 break; 2152 2153 /* allow small jumps within the range */ 2154 if (insn->type == INSN_JUMP_UNCONDITIONAL && 2155 insn->jump_dest && 2156 (insn->jump_dest->offset <= insn->offset || 2157 insn->jump_dest->offset > orig_insn->offset)) 2158 break; 2159 2160 table_reloc = arch_find_switch_table(file, insn); 2161 if (!table_reloc) 2162 continue; 2163 dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc)); 2164 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func) 2165 continue; 2166 2167 return table_reloc; 2168 } 2169 2170 return NULL; 2171 } 2172 2173 /* 2174 * First pass: Mark the head of each jump table so that in the next pass, 2175 * we know when a given jump table ends and the next one starts. 2176 */ 2177 static void mark_func_jump_tables(struct objtool_file *file, 2178 struct symbol *func) 2179 { 2180 struct instruction *insn, *last = NULL; 2181 struct reloc *reloc; 2182 2183 func_for_each_insn(file, func, insn) { 2184 if (!last) 2185 last = insn; 2186 2187 /* 2188 * Store back-pointers for unconditional forward jumps such 2189 * that find_jump_table() can back-track using those and 2190 * avoid some potentially confusing code. 2191 */ 2192 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && 2193 insn->offset > last->offset && 2194 insn->jump_dest->offset > insn->offset && 2195 !insn->jump_dest->first_jump_src) { 2196 2197 insn->jump_dest->first_jump_src = insn; 2198 last = insn->jump_dest; 2199 } 2200 2201 if (insn->type != INSN_JUMP_DYNAMIC) 2202 continue; 2203 2204 reloc = find_jump_table(file, func, insn); 2205 if (reloc) 2206 insn->_jump_table = reloc; 2207 } 2208 } 2209 2210 static int add_func_jump_tables(struct objtool_file *file, 2211 struct symbol *func) 2212 { 2213 struct instruction *insn, *insn_t1 = NULL, *insn_t2; 2214 int ret = 0; 2215 2216 func_for_each_insn(file, func, insn) { 2217 if (!insn_jump_table(insn)) 2218 continue; 2219 2220 if (!insn_t1) { 2221 insn_t1 = insn; 2222 continue; 2223 } 2224 2225 insn_t2 = insn; 2226 2227 ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2)); 2228 if (ret) 2229 return ret; 2230 2231 insn_t1 = insn_t2; 2232 } 2233 2234 if (insn_t1) 2235 ret = add_jump_table(file, insn_t1, NULL); 2236 2237 return ret; 2238 } 2239 2240 /* 2241 * For some switch statements, gcc generates a jump table in the .rodata 2242 * section which contains a list of addresses within the function to jump to. 2243 * This finds these jump tables and adds them to the insn->alts lists. 2244 */ 2245 static int add_jump_table_alts(struct objtool_file *file) 2246 { 2247 struct symbol *func; 2248 int ret; 2249 2250 if (!file->rodata) 2251 return 0; 2252 2253 for_each_sym(file, func) { 2254 if (func->type != STT_FUNC) 2255 continue; 2256 2257 mark_func_jump_tables(file, func); 2258 ret = add_func_jump_tables(file, func); 2259 if (ret) 2260 return ret; 2261 } 2262 2263 return 0; 2264 } 2265 2266 static void set_func_state(struct cfi_state *state) 2267 { 2268 state->cfa = initial_func_cfi.cfa; 2269 memcpy(&state->regs, &initial_func_cfi.regs, 2270 CFI_NUM_REGS * sizeof(struct cfi_reg)); 2271 state->stack_size = initial_func_cfi.cfa.offset; 2272 state->type = UNWIND_HINT_TYPE_CALL; 2273 } 2274 2275 static int read_unwind_hints(struct objtool_file *file) 2276 { 2277 struct cfi_state cfi = init_cfi; 2278 struct section *sec; 2279 struct unwind_hint *hint; 2280 struct instruction *insn; 2281 struct reloc *reloc; 2282 unsigned long offset; 2283 int i; 2284 2285 sec = find_section_by_name(file->elf, ".discard.unwind_hints"); 2286 if (!sec) 2287 return 0; 2288 2289 if (!sec->rsec) { 2290 WARN("missing .rela.discard.unwind_hints section"); 2291 return -1; 2292 } 2293 2294 if (sec->sh.sh_size % sizeof(struct unwind_hint)) { 2295 WARN("struct unwind_hint size mismatch"); 2296 return -1; 2297 } 2298 2299 file->hints = true; 2300 2301 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) { 2302 hint = (struct unwind_hint *)sec->data->d_buf + i; 2303 2304 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); 2305 if (!reloc) { 2306 WARN("can't find reloc for unwind_hints[%d]", i); 2307 return -1; 2308 } 2309 2310 if (reloc->sym->type == STT_SECTION) { 2311 offset = reloc_addend(reloc); 2312 } else if (reloc->sym->local_label) { 2313 offset = reloc->sym->offset; 2314 } else { 2315 WARN("unexpected relocation symbol type in %s", sec->rsec->name); 2316 return -1; 2317 } 2318 2319 insn = find_insn(file, reloc->sym->sec, offset); 2320 if (!insn) { 2321 WARN("can't find insn for unwind_hints[%d]", i); 2322 return -1; 2323 } 2324 2325 insn->hint = true; 2326 2327 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) { 2328 insn->cfi = &force_undefined_cfi; 2329 continue; 2330 } 2331 2332 if (hint->type == UNWIND_HINT_TYPE_SAVE) { 2333 insn->hint = false; 2334 insn->save = true; 2335 continue; 2336 } 2337 2338 if (hint->type == UNWIND_HINT_TYPE_RESTORE) { 2339 insn->restore = true; 2340 continue; 2341 } 2342 2343 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 2344 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 2345 2346 if (sym && sym->bind == STB_GLOBAL) { 2347 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { 2348 WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR"); 2349 } 2350 } 2351 } 2352 2353 if (hint->type == UNWIND_HINT_TYPE_FUNC) { 2354 insn->cfi = &func_cfi; 2355 continue; 2356 } 2357 2358 if (insn->cfi) 2359 cfi = *(insn->cfi); 2360 2361 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { 2362 WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg); 2363 return -1; 2364 } 2365 2366 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); 2367 cfi.type = hint->type; 2368 cfi.signal = hint->signal; 2369 2370 insn->cfi = cfi_hash_find_or_add(&cfi); 2371 } 2372 2373 return 0; 2374 } 2375 2376 static int read_noendbr_hints(struct objtool_file *file) 2377 { 2378 struct instruction *insn; 2379 struct section *rsec; 2380 struct reloc *reloc; 2381 2382 rsec = find_section_by_name(file->elf, ".rela.discard.noendbr"); 2383 if (!rsec) 2384 return 0; 2385 2386 for_each_reloc(rsec, reloc) { 2387 insn = find_insn(file, reloc->sym->sec, 2388 reloc->sym->offset + reloc_addend(reloc)); 2389 if (!insn) { 2390 WARN("bad .discard.noendbr entry"); 2391 return -1; 2392 } 2393 2394 insn->noendbr = 1; 2395 } 2396 2397 return 0; 2398 } 2399 2400 static int read_retpoline_hints(struct objtool_file *file) 2401 { 2402 struct section *rsec; 2403 struct instruction *insn; 2404 struct reloc *reloc; 2405 2406 rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe"); 2407 if (!rsec) 2408 return 0; 2409 2410 for_each_reloc(rsec, reloc) { 2411 if (reloc->sym->type != STT_SECTION) { 2412 WARN("unexpected relocation symbol type in %s", rsec->name); 2413 return -1; 2414 } 2415 2416 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); 2417 if (!insn) { 2418 WARN("bad .discard.retpoline_safe entry"); 2419 return -1; 2420 } 2421 2422 if (insn->type != INSN_JUMP_DYNAMIC && 2423 insn->type != INSN_CALL_DYNAMIC && 2424 insn->type != INSN_RETURN && 2425 insn->type != INSN_NOP) { 2426 WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); 2427 return -1; 2428 } 2429 2430 insn->retpoline_safe = true; 2431 } 2432 2433 return 0; 2434 } 2435 2436 static int read_instr_hints(struct objtool_file *file) 2437 { 2438 struct section *rsec; 2439 struct instruction *insn; 2440 struct reloc *reloc; 2441 2442 rsec = find_section_by_name(file->elf, ".rela.discard.instr_end"); 2443 if (!rsec) 2444 return 0; 2445 2446 for_each_reloc(rsec, reloc) { 2447 if (reloc->sym->type != STT_SECTION) { 2448 WARN("unexpected relocation symbol type in %s", rsec->name); 2449 return -1; 2450 } 2451 2452 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); 2453 if (!insn) { 2454 WARN("bad .discard.instr_end entry"); 2455 return -1; 2456 } 2457 2458 insn->instr--; 2459 } 2460 2461 rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin"); 2462 if (!rsec) 2463 return 0; 2464 2465 for_each_reloc(rsec, reloc) { 2466 if (reloc->sym->type != STT_SECTION) { 2467 WARN("unexpected relocation symbol type in %s", rsec->name); 2468 return -1; 2469 } 2470 2471 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); 2472 if (!insn) { 2473 WARN("bad .discard.instr_begin entry"); 2474 return -1; 2475 } 2476 2477 insn->instr++; 2478 } 2479 2480 return 0; 2481 } 2482 2483 static int read_validate_unret_hints(struct objtool_file *file) 2484 { 2485 struct section *rsec; 2486 struct instruction *insn; 2487 struct reloc *reloc; 2488 2489 rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret"); 2490 if (!rsec) 2491 return 0; 2492 2493 for_each_reloc(rsec, reloc) { 2494 if (reloc->sym->type != STT_SECTION) { 2495 WARN("unexpected relocation symbol type in %s", rsec->name); 2496 return -1; 2497 } 2498 2499 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); 2500 if (!insn) { 2501 WARN("bad .discard.instr_end entry"); 2502 return -1; 2503 } 2504 insn->unret = 1; 2505 } 2506 2507 return 0; 2508 } 2509 2510 2511 static int read_intra_function_calls(struct objtool_file *file) 2512 { 2513 struct instruction *insn; 2514 struct section *rsec; 2515 struct reloc *reloc; 2516 2517 rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls"); 2518 if (!rsec) 2519 return 0; 2520 2521 for_each_reloc(rsec, reloc) { 2522 unsigned long dest_off; 2523 2524 if (reloc->sym->type != STT_SECTION) { 2525 WARN("unexpected relocation symbol type in %s", 2526 rsec->name); 2527 return -1; 2528 } 2529 2530 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc)); 2531 if (!insn) { 2532 WARN("bad .discard.intra_function_call entry"); 2533 return -1; 2534 } 2535 2536 if (insn->type != INSN_CALL) { 2537 WARN_INSN(insn, "intra_function_call not a direct call"); 2538 return -1; 2539 } 2540 2541 /* 2542 * Treat intra-function CALLs as JMPs, but with a stack_op. 2543 * See add_call_destinations(), which strips stack_ops from 2544 * normal CALLs. 2545 */ 2546 insn->type = INSN_JUMP_UNCONDITIONAL; 2547 2548 dest_off = arch_jump_destination(insn); 2549 insn->jump_dest = find_insn(file, insn->sec, dest_off); 2550 if (!insn->jump_dest) { 2551 WARN_INSN(insn, "can't find call dest at %s+0x%lx", 2552 insn->sec->name, dest_off); 2553 return -1; 2554 } 2555 } 2556 2557 return 0; 2558 } 2559 2560 /* 2561 * Return true if name matches an instrumentation function, where calls to that 2562 * function from noinstr code can safely be removed, but compilers won't do so. 2563 */ 2564 static bool is_profiling_func(const char *name) 2565 { 2566 /* 2567 * Many compilers cannot disable KCOV with a function attribute. 2568 */ 2569 if (!strncmp(name, "__sanitizer_cov_", 16)) 2570 return true; 2571 2572 /* 2573 * Some compilers currently do not remove __tsan_func_entry/exit nor 2574 * __tsan_atomic_signal_fence (used for barrier instrumentation) with 2575 * the __no_sanitize_thread attribute, remove them. Once the kernel's 2576 * minimum Clang version is 14.0, this can be removed. 2577 */ 2578 if (!strncmp(name, "__tsan_func_", 12) || 2579 !strcmp(name, "__tsan_atomic_signal_fence")) 2580 return true; 2581 2582 return false; 2583 } 2584 2585 static int classify_symbols(struct objtool_file *file) 2586 { 2587 struct symbol *func; 2588 2589 for_each_sym(file, func) { 2590 if (func->type == STT_NOTYPE && strstarts(func->name, ".L")) 2591 func->local_label = true; 2592 2593 if (func->bind != STB_GLOBAL) 2594 continue; 2595 2596 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, 2597 strlen(STATIC_CALL_TRAMP_PREFIX_STR))) 2598 func->static_call_tramp = true; 2599 2600 if (arch_is_retpoline(func)) 2601 func->retpoline_thunk = true; 2602 2603 if (arch_is_rethunk(func)) 2604 func->return_thunk = true; 2605 2606 if (arch_is_embedded_insn(func)) 2607 func->embedded_insn = true; 2608 2609 if (arch_ftrace_match(func->name)) 2610 func->fentry = true; 2611 2612 if (is_profiling_func(func->name)) 2613 func->profiling_func = true; 2614 } 2615 2616 return 0; 2617 } 2618 2619 static void mark_rodata(struct objtool_file *file) 2620 { 2621 struct section *sec; 2622 bool found = false; 2623 2624 /* 2625 * Search for the following rodata sections, each of which can 2626 * potentially contain jump tables: 2627 * 2628 * - .rodata: can contain GCC switch tables 2629 * - .rodata.<func>: same, if -fdata-sections is being used 2630 * - .rodata..c_jump_table: contains C annotated jump tables 2631 * 2632 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2633 */ 2634 for_each_sec(file, sec) { 2635 if (!strncmp(sec->name, ".rodata", 7) && 2636 !strstr(sec->name, ".str1.")) { 2637 sec->rodata = true; 2638 found = true; 2639 } 2640 } 2641 2642 file->rodata = found; 2643 } 2644 2645 static int decode_sections(struct objtool_file *file) 2646 { 2647 int ret; 2648 2649 mark_rodata(file); 2650 2651 ret = init_pv_ops(file); 2652 if (ret) 2653 return ret; 2654 2655 /* 2656 * Must be before add_{jump_call}_destination. 2657 */ 2658 ret = classify_symbols(file); 2659 if (ret) 2660 return ret; 2661 2662 ret = decode_instructions(file); 2663 if (ret) 2664 return ret; 2665 2666 add_ignores(file); 2667 add_uaccess_safe(file); 2668 2669 ret = add_ignore_alternatives(file); 2670 if (ret) 2671 return ret; 2672 2673 /* 2674 * Must be before read_unwind_hints() since that needs insn->noendbr. 2675 */ 2676 ret = read_noendbr_hints(file); 2677 if (ret) 2678 return ret; 2679 2680 /* 2681 * Must be before add_jump_destinations(), which depends on 'func' 2682 * being set for alternatives, to enable proper sibling call detection. 2683 */ 2684 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) { 2685 ret = add_special_section_alts(file); 2686 if (ret) 2687 return ret; 2688 } 2689 2690 ret = add_jump_destinations(file); 2691 if (ret) 2692 return ret; 2693 2694 /* 2695 * Must be before add_call_destination(); it changes INSN_CALL to 2696 * INSN_JUMP. 2697 */ 2698 ret = read_intra_function_calls(file); 2699 if (ret) 2700 return ret; 2701 2702 ret = add_call_destinations(file); 2703 if (ret) 2704 return ret; 2705 2706 /* 2707 * Must be after add_call_destinations() such that it can override 2708 * dead_end_function() marks. 2709 */ 2710 ret = add_dead_ends(file); 2711 if (ret) 2712 return ret; 2713 2714 ret = add_jump_table_alts(file); 2715 if (ret) 2716 return ret; 2717 2718 ret = read_unwind_hints(file); 2719 if (ret) 2720 return ret; 2721 2722 ret = read_retpoline_hints(file); 2723 if (ret) 2724 return ret; 2725 2726 ret = read_instr_hints(file); 2727 if (ret) 2728 return ret; 2729 2730 ret = read_validate_unret_hints(file); 2731 if (ret) 2732 return ret; 2733 2734 return 0; 2735 } 2736 2737 static bool is_special_call(struct instruction *insn) 2738 { 2739 if (insn->type == INSN_CALL) { 2740 struct symbol *dest = insn_call_dest(insn); 2741 2742 if (!dest) 2743 return false; 2744 2745 if (dest->fentry || dest->embedded_insn) 2746 return true; 2747 } 2748 2749 return false; 2750 } 2751 2752 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state) 2753 { 2754 struct cfi_state *cfi = &state->cfi; 2755 int i; 2756 2757 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap) 2758 return true; 2759 2760 if (cfi->cfa.offset != initial_func_cfi.cfa.offset) 2761 return true; 2762 2763 if (cfi->stack_size != initial_func_cfi.cfa.offset) 2764 return true; 2765 2766 for (i = 0; i < CFI_NUM_REGS; i++) { 2767 if (cfi->regs[i].base != initial_func_cfi.regs[i].base || 2768 cfi->regs[i].offset != initial_func_cfi.regs[i].offset) 2769 return true; 2770 } 2771 2772 return false; 2773 } 2774 2775 static bool check_reg_frame_pos(const struct cfi_reg *reg, 2776 int expected_offset) 2777 { 2778 return reg->base == CFI_CFA && 2779 reg->offset == expected_offset; 2780 } 2781 2782 static bool has_valid_stack_frame(struct insn_state *state) 2783 { 2784 struct cfi_state *cfi = &state->cfi; 2785 2786 if (cfi->cfa.base == CFI_BP && 2787 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) && 2788 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8)) 2789 return true; 2790 2791 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP) 2792 return true; 2793 2794 return false; 2795 } 2796 2797 static int update_cfi_state_regs(struct instruction *insn, 2798 struct cfi_state *cfi, 2799 struct stack_op *op) 2800 { 2801 struct cfi_reg *cfa = &cfi->cfa; 2802 2803 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT) 2804 return 0; 2805 2806 /* push */ 2807 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF) 2808 cfa->offset += 8; 2809 2810 /* pop */ 2811 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF) 2812 cfa->offset -= 8; 2813 2814 /* add immediate to sp */ 2815 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD && 2816 op->dest.reg == CFI_SP && op->src.reg == CFI_SP) 2817 cfa->offset -= op->src.offset; 2818 2819 return 0; 2820 } 2821 2822 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset) 2823 { 2824 if (arch_callee_saved_reg(reg) && 2825 cfi->regs[reg].base == CFI_UNDEFINED) { 2826 cfi->regs[reg].base = base; 2827 cfi->regs[reg].offset = offset; 2828 } 2829 } 2830 2831 static void restore_reg(struct cfi_state *cfi, unsigned char reg) 2832 { 2833 cfi->regs[reg].base = initial_func_cfi.regs[reg].base; 2834 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset; 2835 } 2836 2837 /* 2838 * A note about DRAP stack alignment: 2839 * 2840 * GCC has the concept of a DRAP register, which is used to help keep track of 2841 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP 2842 * register. The typical DRAP pattern is: 2843 * 2844 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10 2845 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp 2846 * 41 ff 72 f8 pushq -0x8(%r10) 2847 * 55 push %rbp 2848 * 48 89 e5 mov %rsp,%rbp 2849 * (more pushes) 2850 * 41 52 push %r10 2851 * ... 2852 * 41 5a pop %r10 2853 * (more pops) 2854 * 5d pop %rbp 2855 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2856 * c3 retq 2857 * 2858 * There are some variations in the epilogues, like: 2859 * 2860 * 5b pop %rbx 2861 * 41 5a pop %r10 2862 * 41 5c pop %r12 2863 * 41 5d pop %r13 2864 * 41 5e pop %r14 2865 * c9 leaveq 2866 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2867 * c3 retq 2868 * 2869 * and: 2870 * 2871 * 4c 8b 55 e8 mov -0x18(%rbp),%r10 2872 * 48 8b 5d e0 mov -0x20(%rbp),%rbx 2873 * 4c 8b 65 f0 mov -0x10(%rbp),%r12 2874 * 4c 8b 6d f8 mov -0x8(%rbp),%r13 2875 * c9 leaveq 2876 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2877 * c3 retq 2878 * 2879 * Sometimes r13 is used as the DRAP register, in which case it's saved and 2880 * restored beforehand: 2881 * 2882 * 41 55 push %r13 2883 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13 2884 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp 2885 * ... 2886 * 49 8d 65 f0 lea -0x10(%r13),%rsp 2887 * 41 5d pop %r13 2888 * c3 retq 2889 */ 2890 static int update_cfi_state(struct instruction *insn, 2891 struct instruction *next_insn, 2892 struct cfi_state *cfi, struct stack_op *op) 2893 { 2894 struct cfi_reg *cfa = &cfi->cfa; 2895 struct cfi_reg *regs = cfi->regs; 2896 2897 /* ignore UNWIND_HINT_UNDEFINED regions */ 2898 if (cfi->force_undefined) 2899 return 0; 2900 2901 /* stack operations don't make sense with an undefined CFA */ 2902 if (cfa->base == CFI_UNDEFINED) { 2903 if (insn_func(insn)) { 2904 WARN_INSN(insn, "undefined stack state"); 2905 return -1; 2906 } 2907 return 0; 2908 } 2909 2910 if (cfi->type == UNWIND_HINT_TYPE_REGS || 2911 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) 2912 return update_cfi_state_regs(insn, cfi, op); 2913 2914 switch (op->dest.type) { 2915 2916 case OP_DEST_REG: 2917 switch (op->src.type) { 2918 2919 case OP_SRC_REG: 2920 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP && 2921 cfa->base == CFI_SP && 2922 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) { 2923 2924 /* mov %rsp, %rbp */ 2925 cfa->base = op->dest.reg; 2926 cfi->bp_scratch = false; 2927 } 2928 2929 else if (op->src.reg == CFI_SP && 2930 op->dest.reg == CFI_BP && cfi->drap) { 2931 2932 /* drap: mov %rsp, %rbp */ 2933 regs[CFI_BP].base = CFI_BP; 2934 regs[CFI_BP].offset = -cfi->stack_size; 2935 cfi->bp_scratch = false; 2936 } 2937 2938 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2939 2940 /* 2941 * mov %rsp, %reg 2942 * 2943 * This is needed for the rare case where GCC 2944 * does: 2945 * 2946 * mov %rsp, %rax 2947 * ... 2948 * mov %rax, %rsp 2949 */ 2950 cfi->vals[op->dest.reg].base = CFI_CFA; 2951 cfi->vals[op->dest.reg].offset = -cfi->stack_size; 2952 } 2953 2954 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP && 2955 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) { 2956 2957 /* 2958 * mov %rbp, %rsp 2959 * 2960 * Restore the original stack pointer (Clang). 2961 */ 2962 cfi->stack_size = -cfi->regs[CFI_BP].offset; 2963 } 2964 2965 else if (op->dest.reg == cfa->base) { 2966 2967 /* mov %reg, %rsp */ 2968 if (cfa->base == CFI_SP && 2969 cfi->vals[op->src.reg].base == CFI_CFA) { 2970 2971 /* 2972 * This is needed for the rare case 2973 * where GCC does something dumb like: 2974 * 2975 * lea 0x8(%rsp), %rcx 2976 * ... 2977 * mov %rcx, %rsp 2978 */ 2979 cfa->offset = -cfi->vals[op->src.reg].offset; 2980 cfi->stack_size = cfa->offset; 2981 2982 } else if (cfa->base == CFI_SP && 2983 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2984 cfi->vals[op->src.reg].offset == cfa->offset) { 2985 2986 /* 2987 * Stack swizzle: 2988 * 2989 * 1: mov %rsp, (%[tos]) 2990 * 2: mov %[tos], %rsp 2991 * ... 2992 * 3: pop %rsp 2993 * 2994 * Where: 2995 * 2996 * 1 - places a pointer to the previous 2997 * stack at the Top-of-Stack of the 2998 * new stack. 2999 * 3000 * 2 - switches to the new stack. 3001 * 3002 * 3 - pops the Top-of-Stack to restore 3003 * the original stack. 3004 * 3005 * Note: we set base to SP_INDIRECT 3006 * here and preserve offset. Therefore 3007 * when the unwinder reaches ToS it 3008 * will dereference SP and then add the 3009 * offset to find the next frame, IOW: 3010 * (%rsp) + offset. 3011 */ 3012 cfa->base = CFI_SP_INDIRECT; 3013 3014 } else { 3015 cfa->base = CFI_UNDEFINED; 3016 cfa->offset = 0; 3017 } 3018 } 3019 3020 else if (op->dest.reg == CFI_SP && 3021 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 3022 cfi->vals[op->src.reg].offset == cfa->offset) { 3023 3024 /* 3025 * The same stack swizzle case 2) as above. But 3026 * because we can't change cfa->base, case 3) 3027 * will become a regular POP. Pretend we're a 3028 * PUSH so things don't go unbalanced. 3029 */ 3030 cfi->stack_size += 8; 3031 } 3032 3033 3034 break; 3035 3036 case OP_SRC_ADD: 3037 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) { 3038 3039 /* add imm, %rsp */ 3040 cfi->stack_size -= op->src.offset; 3041 if (cfa->base == CFI_SP) 3042 cfa->offset -= op->src.offset; 3043 break; 3044 } 3045 3046 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP && 3047 insn->sym->frame_pointer) { 3048 /* addi.d fp,sp,imm on LoongArch */ 3049 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) { 3050 cfa->base = CFI_BP; 3051 cfa->offset = 0; 3052 } 3053 break; 3054 } 3055 3056 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { 3057 /* addi.d sp,fp,imm on LoongArch */ 3058 if (cfa->base == CFI_BP && cfa->offset == 0) { 3059 if (insn->sym->frame_pointer) { 3060 cfa->base = CFI_SP; 3061 cfa->offset = -op->src.offset; 3062 } 3063 } else { 3064 /* lea disp(%rbp), %rsp */ 3065 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); 3066 } 3067 break; 3068 } 3069 3070 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 3071 3072 /* drap: lea disp(%rsp), %drap */ 3073 cfi->drap_reg = op->dest.reg; 3074 3075 /* 3076 * lea disp(%rsp), %reg 3077 * 3078 * This is needed for the rare case where GCC 3079 * does something dumb like: 3080 * 3081 * lea 0x8(%rsp), %rcx 3082 * ... 3083 * mov %rcx, %rsp 3084 */ 3085 cfi->vals[op->dest.reg].base = CFI_CFA; 3086 cfi->vals[op->dest.reg].offset = \ 3087 -cfi->stack_size + op->src.offset; 3088 3089 break; 3090 } 3091 3092 if (cfi->drap && op->dest.reg == CFI_SP && 3093 op->src.reg == cfi->drap_reg) { 3094 3095 /* drap: lea disp(%drap), %rsp */ 3096 cfa->base = CFI_SP; 3097 cfa->offset = cfi->stack_size = -op->src.offset; 3098 cfi->drap_reg = CFI_UNDEFINED; 3099 cfi->drap = false; 3100 break; 3101 } 3102 3103 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { 3104 WARN_INSN(insn, "unsupported stack register modification"); 3105 return -1; 3106 } 3107 3108 break; 3109 3110 case OP_SRC_AND: 3111 if (op->dest.reg != CFI_SP || 3112 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || 3113 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { 3114 WARN_INSN(insn, "unsupported stack pointer realignment"); 3115 return -1; 3116 } 3117 3118 if (cfi->drap_reg != CFI_UNDEFINED) { 3119 /* drap: and imm, %rsp */ 3120 cfa->base = cfi->drap_reg; 3121 cfa->offset = cfi->stack_size = 0; 3122 cfi->drap = true; 3123 } 3124 3125 /* 3126 * Older versions of GCC (4.8ish) realign the stack 3127 * without DRAP, with a frame pointer. 3128 */ 3129 3130 break; 3131 3132 case OP_SRC_POP: 3133 case OP_SRC_POPF: 3134 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) { 3135 3136 /* pop %rsp; # restore from a stack swizzle */ 3137 cfa->base = CFI_SP; 3138 break; 3139 } 3140 3141 if (!cfi->drap && op->dest.reg == cfa->base) { 3142 3143 /* pop %rbp */ 3144 cfa->base = CFI_SP; 3145 } 3146 3147 if (cfi->drap && cfa->base == CFI_BP_INDIRECT && 3148 op->dest.reg == cfi->drap_reg && 3149 cfi->drap_offset == -cfi->stack_size) { 3150 3151 /* drap: pop %drap */ 3152 cfa->base = cfi->drap_reg; 3153 cfa->offset = 0; 3154 cfi->drap_offset = -1; 3155 3156 } else if (cfi->stack_size == -regs[op->dest.reg].offset) { 3157 3158 /* pop %reg */ 3159 restore_reg(cfi, op->dest.reg); 3160 } 3161 3162 cfi->stack_size -= 8; 3163 if (cfa->base == CFI_SP) 3164 cfa->offset -= 8; 3165 3166 break; 3167 3168 case OP_SRC_REG_INDIRECT: 3169 if (!cfi->drap && op->dest.reg == cfa->base && 3170 op->dest.reg == CFI_BP) { 3171 3172 /* mov disp(%rsp), %rbp */ 3173 cfa->base = CFI_SP; 3174 cfa->offset = cfi->stack_size; 3175 } 3176 3177 if (cfi->drap && op->src.reg == CFI_BP && 3178 op->src.offset == cfi->drap_offset) { 3179 3180 /* drap: mov disp(%rbp), %drap */ 3181 cfa->base = cfi->drap_reg; 3182 cfa->offset = 0; 3183 cfi->drap_offset = -1; 3184 } 3185 3186 if (cfi->drap && op->src.reg == CFI_BP && 3187 op->src.offset == regs[op->dest.reg].offset) { 3188 3189 /* drap: mov disp(%rbp), %reg */ 3190 restore_reg(cfi, op->dest.reg); 3191 3192 } else if (op->src.reg == cfa->base && 3193 op->src.offset == regs[op->dest.reg].offset + cfa->offset) { 3194 3195 /* mov disp(%rbp), %reg */ 3196 /* mov disp(%rsp), %reg */ 3197 restore_reg(cfi, op->dest.reg); 3198 3199 } else if (op->src.reg == CFI_SP && 3200 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) { 3201 3202 /* mov disp(%rsp), %reg */ 3203 restore_reg(cfi, op->dest.reg); 3204 } 3205 3206 break; 3207 3208 default: 3209 WARN_INSN(insn, "unknown stack-related instruction"); 3210 return -1; 3211 } 3212 3213 break; 3214 3215 case OP_DEST_PUSH: 3216 case OP_DEST_PUSHF: 3217 cfi->stack_size += 8; 3218 if (cfa->base == CFI_SP) 3219 cfa->offset += 8; 3220 3221 if (op->src.type != OP_SRC_REG) 3222 break; 3223 3224 if (cfi->drap) { 3225 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3226 3227 /* drap: push %drap */ 3228 cfa->base = CFI_BP_INDIRECT; 3229 cfa->offset = -cfi->stack_size; 3230 3231 /* save drap so we know when to restore it */ 3232 cfi->drap_offset = -cfi->stack_size; 3233 3234 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) { 3235 3236 /* drap: push %rbp */ 3237 cfi->stack_size = 0; 3238 3239 } else { 3240 3241 /* drap: push %reg */ 3242 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); 3243 } 3244 3245 } else { 3246 3247 /* push %reg */ 3248 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size); 3249 } 3250 3251 /* detect when asm code uses rbp as a scratch register */ 3252 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP && 3253 cfa->base != CFI_BP) 3254 cfi->bp_scratch = true; 3255 break; 3256 3257 case OP_DEST_REG_INDIRECT: 3258 3259 if (cfi->drap) { 3260 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3261 3262 /* drap: mov %drap, disp(%rbp) */ 3263 cfa->base = CFI_BP_INDIRECT; 3264 cfa->offset = op->dest.offset; 3265 3266 /* save drap offset so we know when to restore it */ 3267 cfi->drap_offset = op->dest.offset; 3268 } else { 3269 3270 /* drap: mov reg, disp(%rbp) */ 3271 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); 3272 } 3273 3274 } else if (op->dest.reg == cfa->base) { 3275 3276 /* mov reg, disp(%rbp) */ 3277 /* mov reg, disp(%rsp) */ 3278 save_reg(cfi, op->src.reg, CFI_CFA, 3279 op->dest.offset - cfi->cfa.offset); 3280 3281 } else if (op->dest.reg == CFI_SP) { 3282 3283 /* mov reg, disp(%rsp) */ 3284 save_reg(cfi, op->src.reg, CFI_CFA, 3285 op->dest.offset - cfi->stack_size); 3286 3287 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) { 3288 3289 /* mov %rsp, (%reg); # setup a stack swizzle. */ 3290 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT; 3291 cfi->vals[op->dest.reg].offset = cfa->offset; 3292 } 3293 3294 break; 3295 3296 case OP_DEST_MEM: 3297 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { 3298 WARN_INSN(insn, "unknown stack-related memory operation"); 3299 return -1; 3300 } 3301 3302 /* pop mem */ 3303 cfi->stack_size -= 8; 3304 if (cfa->base == CFI_SP) 3305 cfa->offset -= 8; 3306 3307 break; 3308 3309 default: 3310 WARN_INSN(insn, "unknown stack-related instruction"); 3311 return -1; 3312 } 3313 3314 return 0; 3315 } 3316 3317 /* 3318 * The stack layouts of alternatives instructions can sometimes diverge when 3319 * they have stack modifications. That's fine as long as the potential stack 3320 * layouts don't conflict at any given potential instruction boundary. 3321 * 3322 * Flatten the CFIs of the different alternative code streams (both original 3323 * and replacement) into a single shared CFI array which can be used to detect 3324 * conflicts and nicely feed a linear array of ORC entries to the unwinder. 3325 */ 3326 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn) 3327 { 3328 struct cfi_state **alt_cfi; 3329 int group_off; 3330 3331 if (!insn->alt_group) 3332 return 0; 3333 3334 if (!insn->cfi) { 3335 WARN("CFI missing"); 3336 return -1; 3337 } 3338 3339 alt_cfi = insn->alt_group->cfi; 3340 group_off = insn->offset - insn->alt_group->first_insn->offset; 3341 3342 if (!alt_cfi[group_off]) { 3343 alt_cfi[group_off] = insn->cfi; 3344 } else { 3345 if (cficmp(alt_cfi[group_off], insn->cfi)) { 3346 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group; 3347 struct instruction *orig = orig_group->first_insn; 3348 char *where = offstr(insn->sec, insn->offset); 3349 WARN_INSN(orig, "stack layout conflict in alternatives: %s", where); 3350 free(where); 3351 return -1; 3352 } 3353 } 3354 3355 return 0; 3356 } 3357 3358 static int handle_insn_ops(struct instruction *insn, 3359 struct instruction *next_insn, 3360 struct insn_state *state) 3361 { 3362 struct stack_op *op; 3363 3364 for (op = insn->stack_ops; op; op = op->next) { 3365 3366 if (update_cfi_state(insn, next_insn, &state->cfi, op)) 3367 return 1; 3368 3369 if (!insn->alt_group) 3370 continue; 3371 3372 if (op->dest.type == OP_DEST_PUSHF) { 3373 if (!state->uaccess_stack) { 3374 state->uaccess_stack = 1; 3375 } else if (state->uaccess_stack >> 31) { 3376 WARN_INSN(insn, "PUSHF stack exhausted"); 3377 return 1; 3378 } 3379 state->uaccess_stack <<= 1; 3380 state->uaccess_stack |= state->uaccess; 3381 } 3382 3383 if (op->src.type == OP_SRC_POPF) { 3384 if (state->uaccess_stack) { 3385 state->uaccess = state->uaccess_stack & 1; 3386 state->uaccess_stack >>= 1; 3387 if (state->uaccess_stack == 1) 3388 state->uaccess_stack = 0; 3389 } 3390 } 3391 } 3392 3393 return 0; 3394 } 3395 3396 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) 3397 { 3398 struct cfi_state *cfi1 = insn->cfi; 3399 int i; 3400 3401 if (!cfi1) { 3402 WARN("CFI missing"); 3403 return false; 3404 } 3405 3406 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { 3407 3408 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d", 3409 cfi1->cfa.base, cfi1->cfa.offset, 3410 cfi2->cfa.base, cfi2->cfa.offset); 3411 3412 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) { 3413 for (i = 0; i < CFI_NUM_REGS; i++) { 3414 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], 3415 sizeof(struct cfi_reg))) 3416 continue; 3417 3418 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", 3419 i, cfi1->regs[i].base, cfi1->regs[i].offset, 3420 i, cfi2->regs[i].base, cfi2->regs[i].offset); 3421 break; 3422 } 3423 3424 } else if (cfi1->type != cfi2->type) { 3425 3426 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d", 3427 cfi1->type, cfi2->type); 3428 3429 } else if (cfi1->drap != cfi2->drap || 3430 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || 3431 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { 3432 3433 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", 3434 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, 3435 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); 3436 3437 } else 3438 return true; 3439 3440 return false; 3441 } 3442 3443 static inline bool func_uaccess_safe(struct symbol *func) 3444 { 3445 if (func) 3446 return func->uaccess_safe; 3447 3448 return false; 3449 } 3450 3451 static inline const char *call_dest_name(struct instruction *insn) 3452 { 3453 static char pvname[19]; 3454 struct reloc *reloc; 3455 int idx; 3456 3457 if (insn_call_dest(insn)) 3458 return insn_call_dest(insn)->name; 3459 3460 reloc = insn_reloc(NULL, insn); 3461 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) { 3462 idx = (reloc_addend(reloc) / sizeof(void *)); 3463 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx); 3464 return pvname; 3465 } 3466 3467 return "{dynamic}"; 3468 } 3469 3470 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn) 3471 { 3472 struct symbol *target; 3473 struct reloc *reloc; 3474 int idx; 3475 3476 reloc = insn_reloc(file, insn); 3477 if (!reloc || strcmp(reloc->sym->name, "pv_ops")) 3478 return false; 3479 3480 idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *)); 3481 3482 if (file->pv_ops[idx].clean) 3483 return true; 3484 3485 file->pv_ops[idx].clean = true; 3486 3487 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) { 3488 if (!target->sec->noinstr) { 3489 WARN("pv_ops[%d]: %s", idx, target->name); 3490 file->pv_ops[idx].clean = false; 3491 } 3492 } 3493 3494 return file->pv_ops[idx].clean; 3495 } 3496 3497 static inline bool noinstr_call_dest(struct objtool_file *file, 3498 struct instruction *insn, 3499 struct symbol *func) 3500 { 3501 /* 3502 * We can't deal with indirect function calls at present; 3503 * assume they're instrumented. 3504 */ 3505 if (!func) { 3506 if (file->pv_ops) 3507 return pv_call_dest(file, insn); 3508 3509 return false; 3510 } 3511 3512 /* 3513 * If the symbol is from a noinstr section; we good. 3514 */ 3515 if (func->sec->noinstr) 3516 return true; 3517 3518 /* 3519 * If the symbol is a static_call trampoline, we can't tell. 3520 */ 3521 if (func->static_call_tramp) 3522 return true; 3523 3524 /* 3525 * The __ubsan_handle_*() calls are like WARN(), they only happen when 3526 * something 'BAD' happened. At the risk of taking the machine down, 3527 * let them proceed to get the message out. 3528 */ 3529 if (!strncmp(func->name, "__ubsan_handle_", 15)) 3530 return true; 3531 3532 return false; 3533 } 3534 3535 static int validate_call(struct objtool_file *file, 3536 struct instruction *insn, 3537 struct insn_state *state) 3538 { 3539 if (state->noinstr && state->instr <= 0 && 3540 !noinstr_call_dest(file, insn, insn_call_dest(insn))) { 3541 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn)); 3542 return 1; 3543 } 3544 3545 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) { 3546 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn)); 3547 return 1; 3548 } 3549 3550 if (state->df) { 3551 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn)); 3552 return 1; 3553 } 3554 3555 return 0; 3556 } 3557 3558 static int validate_sibling_call(struct objtool_file *file, 3559 struct instruction *insn, 3560 struct insn_state *state) 3561 { 3562 if (insn_func(insn) && has_modified_stack_frame(insn, state)) { 3563 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame"); 3564 return 1; 3565 } 3566 3567 return validate_call(file, insn, state); 3568 } 3569 3570 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 3571 { 3572 if (state->noinstr && state->instr > 0) { 3573 WARN_INSN(insn, "return with instrumentation enabled"); 3574 return 1; 3575 } 3576 3577 if (state->uaccess && !func_uaccess_safe(func)) { 3578 WARN_INSN(insn, "return with UACCESS enabled"); 3579 return 1; 3580 } 3581 3582 if (!state->uaccess && func_uaccess_safe(func)) { 3583 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function"); 3584 return 1; 3585 } 3586 3587 if (state->df) { 3588 WARN_INSN(insn, "return with DF set"); 3589 return 1; 3590 } 3591 3592 if (func && has_modified_stack_frame(insn, state)) { 3593 WARN_INSN(insn, "return with modified stack frame"); 3594 return 1; 3595 } 3596 3597 if (state->cfi.bp_scratch) { 3598 WARN_INSN(insn, "BP used as a scratch register"); 3599 return 1; 3600 } 3601 3602 return 0; 3603 } 3604 3605 static struct instruction *next_insn_to_validate(struct objtool_file *file, 3606 struct instruction *insn) 3607 { 3608 struct alt_group *alt_group = insn->alt_group; 3609 3610 /* 3611 * Simulate the fact that alternatives are patched in-place. When the 3612 * end of a replacement alt_group is reached, redirect objtool flow to 3613 * the end of the original alt_group. 3614 * 3615 * insn->alts->insn -> alt_group->first_insn 3616 * ... 3617 * alt_group->last_insn 3618 * [alt_group->nop] -> next(orig_group->last_insn) 3619 */ 3620 if (alt_group) { 3621 if (alt_group->nop) { 3622 /* ->nop implies ->orig_group */ 3623 if (insn == alt_group->last_insn) 3624 return alt_group->nop; 3625 if (insn == alt_group->nop) 3626 goto next_orig; 3627 } 3628 if (insn == alt_group->last_insn && alt_group->orig_group) 3629 goto next_orig; 3630 } 3631 3632 return next_insn_same_sec(file, insn); 3633 3634 next_orig: 3635 return next_insn_same_sec(file, alt_group->orig_group->last_insn); 3636 } 3637 3638 /* 3639 * Follow the branch starting at the given instruction, and recursively follow 3640 * any other branches (jumps). Meanwhile, track the frame pointer state at 3641 * each instruction and validate all the rules described in 3642 * tools/objtool/Documentation/objtool.txt. 3643 */ 3644 static int validate_branch(struct objtool_file *file, struct symbol *func, 3645 struct instruction *insn, struct insn_state state) 3646 { 3647 struct alternative *alt; 3648 struct instruction *next_insn, *prev_insn = NULL; 3649 struct section *sec; 3650 u8 visited; 3651 int ret; 3652 3653 sec = insn->sec; 3654 3655 while (1) { 3656 next_insn = next_insn_to_validate(file, insn); 3657 3658 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { 3659 /* Ignore KCFI type preambles, which always fall through */ 3660 if (!strncmp(func->name, "__cfi_", 6) || 3661 !strncmp(func->name, "__pfx_", 6)) 3662 return 0; 3663 3664 WARN("%s() falls through to next function %s()", 3665 func->name, insn_func(insn)->name); 3666 return 1; 3667 } 3668 3669 if (func && insn->ignore) { 3670 WARN_INSN(insn, "BUG: why am I validating an ignored function?"); 3671 return 1; 3672 } 3673 3674 visited = VISITED_BRANCH << state.uaccess; 3675 if (insn->visited & VISITED_BRANCH_MASK) { 3676 if (!insn->hint && !insn_cfi_match(insn, &state.cfi)) 3677 return 1; 3678 3679 if (insn->visited & visited) 3680 return 0; 3681 } else { 3682 nr_insns_visited++; 3683 } 3684 3685 if (state.noinstr) 3686 state.instr += insn->instr; 3687 3688 if (insn->hint) { 3689 if (insn->restore) { 3690 struct instruction *save_insn, *i; 3691 3692 i = insn; 3693 save_insn = NULL; 3694 3695 sym_for_each_insn_continue_reverse(file, func, i) { 3696 if (i->save) { 3697 save_insn = i; 3698 break; 3699 } 3700 } 3701 3702 if (!save_insn) { 3703 WARN_INSN(insn, "no corresponding CFI save for CFI restore"); 3704 return 1; 3705 } 3706 3707 if (!save_insn->visited) { 3708 /* 3709 * If the restore hint insn is at the 3710 * beginning of a basic block and was 3711 * branched to from elsewhere, and the 3712 * save insn hasn't been visited yet, 3713 * defer following this branch for now. 3714 * It will be seen later via the 3715 * straight-line path. 3716 */ 3717 if (!prev_insn) 3718 return 0; 3719 3720 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo"); 3721 return 1; 3722 } 3723 3724 insn->cfi = save_insn->cfi; 3725 nr_cfi_reused++; 3726 } 3727 3728 state.cfi = *insn->cfi; 3729 } else { 3730 /* XXX track if we actually changed state.cfi */ 3731 3732 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) { 3733 insn->cfi = prev_insn->cfi; 3734 nr_cfi_reused++; 3735 } else { 3736 insn->cfi = cfi_hash_find_or_add(&state.cfi); 3737 } 3738 } 3739 3740 insn->visited |= visited; 3741 3742 if (propagate_alt_cfi(file, insn)) 3743 return 1; 3744 3745 if (!insn->ignore_alts && insn->alts) { 3746 bool skip_orig = false; 3747 3748 for (alt = insn->alts; alt; alt = alt->next) { 3749 if (alt->skip_orig) 3750 skip_orig = true; 3751 3752 ret = validate_branch(file, func, alt->insn, state); 3753 if (ret) { 3754 BT_INSN(insn, "(alt)"); 3755 return ret; 3756 } 3757 } 3758 3759 if (skip_orig) 3760 return 0; 3761 } 3762 3763 if (handle_insn_ops(insn, next_insn, &state)) 3764 return 1; 3765 3766 switch (insn->type) { 3767 3768 case INSN_RETURN: 3769 return validate_return(func, insn, &state); 3770 3771 case INSN_CALL: 3772 case INSN_CALL_DYNAMIC: 3773 ret = validate_call(file, insn, &state); 3774 if (ret) 3775 return ret; 3776 3777 if (opts.stackval && func && !is_special_call(insn) && 3778 !has_valid_stack_frame(&state)) { 3779 WARN_INSN(insn, "call without frame pointer save/setup"); 3780 return 1; 3781 } 3782 3783 if (insn->dead_end) 3784 return 0; 3785 3786 break; 3787 3788 case INSN_JUMP_CONDITIONAL: 3789 case INSN_JUMP_UNCONDITIONAL: 3790 if (is_sibling_call(insn)) { 3791 ret = validate_sibling_call(file, insn, &state); 3792 if (ret) 3793 return ret; 3794 3795 } else if (insn->jump_dest) { 3796 ret = validate_branch(file, func, 3797 insn->jump_dest, state); 3798 if (ret) { 3799 BT_INSN(insn, "(branch)"); 3800 return ret; 3801 } 3802 } 3803 3804 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3805 return 0; 3806 3807 break; 3808 3809 case INSN_JUMP_DYNAMIC: 3810 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3811 if (is_sibling_call(insn)) { 3812 ret = validate_sibling_call(file, insn, &state); 3813 if (ret) 3814 return ret; 3815 } 3816 3817 if (insn->type == INSN_JUMP_DYNAMIC) 3818 return 0; 3819 3820 break; 3821 3822 case INSN_CONTEXT_SWITCH: 3823 if (func && (!next_insn || !next_insn->hint)) { 3824 WARN_INSN(insn, "unsupported instruction in callable function"); 3825 return 1; 3826 } 3827 return 0; 3828 3829 case INSN_STAC: 3830 if (state.uaccess) { 3831 WARN_INSN(insn, "recursive UACCESS enable"); 3832 return 1; 3833 } 3834 3835 state.uaccess = true; 3836 break; 3837 3838 case INSN_CLAC: 3839 if (!state.uaccess && func) { 3840 WARN_INSN(insn, "redundant UACCESS disable"); 3841 return 1; 3842 } 3843 3844 if (func_uaccess_safe(func) && !state.uaccess_stack) { 3845 WARN_INSN(insn, "UACCESS-safe disables UACCESS"); 3846 return 1; 3847 } 3848 3849 state.uaccess = false; 3850 break; 3851 3852 case INSN_STD: 3853 if (state.df) { 3854 WARN_INSN(insn, "recursive STD"); 3855 return 1; 3856 } 3857 3858 state.df = true; 3859 break; 3860 3861 case INSN_CLD: 3862 if (!state.df && func) { 3863 WARN_INSN(insn, "redundant CLD"); 3864 return 1; 3865 } 3866 3867 state.df = false; 3868 break; 3869 3870 default: 3871 break; 3872 } 3873 3874 if (insn->dead_end) 3875 return 0; 3876 3877 if (!next_insn) { 3878 if (state.cfi.cfa.base == CFI_UNDEFINED) 3879 return 0; 3880 WARN("%s: unexpected end of section", sec->name); 3881 return 1; 3882 } 3883 3884 prev_insn = insn; 3885 insn = next_insn; 3886 } 3887 3888 return 0; 3889 } 3890 3891 static int validate_unwind_hint(struct objtool_file *file, 3892 struct instruction *insn, 3893 struct insn_state *state) 3894 { 3895 if (insn->hint && !insn->visited && !insn->ignore) { 3896 int ret = validate_branch(file, insn_func(insn), insn, *state); 3897 if (ret) 3898 BT_INSN(insn, "<=== (hint)"); 3899 return ret; 3900 } 3901 3902 return 0; 3903 } 3904 3905 static int validate_unwind_hints(struct objtool_file *file, struct section *sec) 3906 { 3907 struct instruction *insn; 3908 struct insn_state state; 3909 int warnings = 0; 3910 3911 if (!file->hints) 3912 return 0; 3913 3914 init_insn_state(file, &state, sec); 3915 3916 if (sec) { 3917 sec_for_each_insn(file, sec, insn) 3918 warnings += validate_unwind_hint(file, insn, &state); 3919 } else { 3920 for_each_insn(file, insn) 3921 warnings += validate_unwind_hint(file, insn, &state); 3922 } 3923 3924 return warnings; 3925 } 3926 3927 /* 3928 * Validate rethunk entry constraint: must untrain RET before the first RET. 3929 * 3930 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes 3931 * before an actual RET instruction. 3932 */ 3933 static int validate_unret(struct objtool_file *file, struct instruction *insn) 3934 { 3935 struct instruction *next, *dest; 3936 int ret; 3937 3938 for (;;) { 3939 next = next_insn_to_validate(file, insn); 3940 3941 if (insn->visited & VISITED_UNRET) 3942 return 0; 3943 3944 insn->visited |= VISITED_UNRET; 3945 3946 if (!insn->ignore_alts && insn->alts) { 3947 struct alternative *alt; 3948 bool skip_orig = false; 3949 3950 for (alt = insn->alts; alt; alt = alt->next) { 3951 if (alt->skip_orig) 3952 skip_orig = true; 3953 3954 ret = validate_unret(file, alt->insn); 3955 if (ret) { 3956 BT_INSN(insn, "(alt)"); 3957 return ret; 3958 } 3959 } 3960 3961 if (skip_orig) 3962 return 0; 3963 } 3964 3965 switch (insn->type) { 3966 3967 case INSN_CALL_DYNAMIC: 3968 case INSN_JUMP_DYNAMIC: 3969 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3970 WARN_INSN(insn, "early indirect call"); 3971 return 1; 3972 3973 case INSN_JUMP_UNCONDITIONAL: 3974 case INSN_JUMP_CONDITIONAL: 3975 if (!is_sibling_call(insn)) { 3976 if (!insn->jump_dest) { 3977 WARN_INSN(insn, "unresolved jump target after linking?!?"); 3978 return -1; 3979 } 3980 ret = validate_unret(file, insn->jump_dest); 3981 if (ret) { 3982 BT_INSN(insn, "(branch%s)", 3983 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : ""); 3984 return ret; 3985 } 3986 3987 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3988 return 0; 3989 3990 break; 3991 } 3992 3993 /* fallthrough */ 3994 case INSN_CALL: 3995 dest = find_insn(file, insn_call_dest(insn)->sec, 3996 insn_call_dest(insn)->offset); 3997 if (!dest) { 3998 WARN("Unresolved function after linking!?: %s", 3999 insn_call_dest(insn)->name); 4000 return -1; 4001 } 4002 4003 ret = validate_unret(file, dest); 4004 if (ret) { 4005 BT_INSN(insn, "(call)"); 4006 return ret; 4007 } 4008 /* 4009 * If a call returns without error, it must have seen UNTRAIN_RET. 4010 * Therefore any non-error return is a success. 4011 */ 4012 return 0; 4013 4014 case INSN_RETURN: 4015 WARN_INSN(insn, "RET before UNTRAIN"); 4016 return 1; 4017 4018 case INSN_NOP: 4019 if (insn->retpoline_safe) 4020 return 0; 4021 break; 4022 4023 default: 4024 break; 4025 } 4026 4027 if (!next) { 4028 WARN_INSN(insn, "teh end!"); 4029 return -1; 4030 } 4031 insn = next; 4032 } 4033 4034 return 0; 4035 } 4036 4037 /* 4038 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter 4039 * VALIDATE_UNRET_END before RET. 4040 */ 4041 static int validate_unrets(struct objtool_file *file) 4042 { 4043 struct instruction *insn; 4044 int ret, warnings = 0; 4045 4046 for_each_insn(file, insn) { 4047 if (!insn->unret) 4048 continue; 4049 4050 ret = validate_unret(file, insn); 4051 if (ret < 0) { 4052 WARN_INSN(insn, "Failed UNRET validation"); 4053 return ret; 4054 } 4055 warnings += ret; 4056 } 4057 4058 return warnings; 4059 } 4060 4061 static int validate_retpoline(struct objtool_file *file) 4062 { 4063 struct instruction *insn; 4064 int warnings = 0; 4065 4066 for_each_insn(file, insn) { 4067 if (insn->type != INSN_JUMP_DYNAMIC && 4068 insn->type != INSN_CALL_DYNAMIC && 4069 insn->type != INSN_RETURN) 4070 continue; 4071 4072 if (insn->retpoline_safe) 4073 continue; 4074 4075 if (insn->sec->init) 4076 continue; 4077 4078 if (insn->type == INSN_RETURN) { 4079 if (opts.rethunk) { 4080 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build"); 4081 } else 4082 continue; 4083 } else { 4084 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build", 4085 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); 4086 } 4087 4088 warnings++; 4089 } 4090 4091 return warnings; 4092 } 4093 4094 static bool is_kasan_insn(struct instruction *insn) 4095 { 4096 return (insn->type == INSN_CALL && 4097 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return")); 4098 } 4099 4100 static bool is_ubsan_insn(struct instruction *insn) 4101 { 4102 return (insn->type == INSN_CALL && 4103 !strcmp(insn_call_dest(insn)->name, 4104 "__ubsan_handle_builtin_unreachable")); 4105 } 4106 4107 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) 4108 { 4109 int i; 4110 struct instruction *prev_insn; 4111 4112 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP) 4113 return true; 4114 4115 /* 4116 * Ignore alternative replacement instructions. This can happen 4117 * when a whitelisted function uses one of the ALTERNATIVE macros. 4118 */ 4119 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 4120 !strcmp(insn->sec->name, ".altinstr_aux")) 4121 return true; 4122 4123 /* 4124 * Whole archive runs might encounter dead code from weak symbols. 4125 * This is where the linker will have dropped the weak symbol in 4126 * favour of a regular symbol, but leaves the code in place. 4127 * 4128 * In this case we'll find a piece of code (whole function) that is not 4129 * covered by a !section symbol. Ignore them. 4130 */ 4131 if (opts.link && !insn_func(insn)) { 4132 int size = find_symbol_hole_containing(insn->sec, insn->offset); 4133 unsigned long end = insn->offset + size; 4134 4135 if (!size) /* not a hole */ 4136 return false; 4137 4138 if (size < 0) /* hole until the end */ 4139 return true; 4140 4141 sec_for_each_insn_continue(file, insn) { 4142 /* 4143 * If we reach a visited instruction at or before the 4144 * end of the hole, ignore the unreachable. 4145 */ 4146 if (insn->visited) 4147 return true; 4148 4149 if (insn->offset >= end) 4150 break; 4151 4152 /* 4153 * If this hole jumps to a .cold function, mark it ignore too. 4154 */ 4155 if (insn->jump_dest && insn_func(insn->jump_dest) && 4156 strstr(insn_func(insn->jump_dest)->name, ".cold")) { 4157 struct instruction *dest = insn->jump_dest; 4158 func_for_each_insn(file, insn_func(dest), dest) 4159 dest->ignore = true; 4160 } 4161 } 4162 4163 return false; 4164 } 4165 4166 if (!insn_func(insn)) 4167 return false; 4168 4169 if (insn_func(insn)->static_call_tramp) 4170 return true; 4171 4172 /* 4173 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees 4174 * __builtin_unreachable(). The BUG() macro has an unreachable() after 4175 * the UD2, which causes GCC's undefined trap logic to emit another UD2 4176 * (or occasionally a JMP to UD2). 4177 * 4178 * It may also insert a UD2 after calling a __noreturn function. 4179 */ 4180 prev_insn = prev_insn_same_sec(file, insn); 4181 if (prev_insn->dead_end && 4182 (insn->type == INSN_BUG || 4183 (insn->type == INSN_JUMP_UNCONDITIONAL && 4184 insn->jump_dest && insn->jump_dest->type == INSN_BUG))) 4185 return true; 4186 4187 /* 4188 * Check if this (or a subsequent) instruction is related to 4189 * CONFIG_UBSAN or CONFIG_KASAN. 4190 * 4191 * End the search at 5 instructions to avoid going into the weeds. 4192 */ 4193 for (i = 0; i < 5; i++) { 4194 4195 if (is_kasan_insn(insn) || is_ubsan_insn(insn)) 4196 return true; 4197 4198 if (insn->type == INSN_JUMP_UNCONDITIONAL) { 4199 if (insn->jump_dest && 4200 insn_func(insn->jump_dest) == insn_func(insn)) { 4201 insn = insn->jump_dest; 4202 continue; 4203 } 4204 4205 break; 4206 } 4207 4208 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len) 4209 break; 4210 4211 insn = next_insn_same_sec(file, insn); 4212 } 4213 4214 return false; 4215 } 4216 4217 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func) 4218 { 4219 struct instruction *insn, *prev; 4220 struct cfi_state *cfi; 4221 4222 insn = find_insn(file, func->sec, func->offset); 4223 if (!insn) 4224 return -1; 4225 4226 for (prev = prev_insn_same_sec(file, insn); 4227 prev; 4228 prev = prev_insn_same_sec(file, prev)) { 4229 u64 offset; 4230 4231 if (prev->type != INSN_NOP) 4232 return -1; 4233 4234 offset = func->offset - prev->offset; 4235 4236 if (offset > opts.prefix) 4237 return -1; 4238 4239 if (offset < opts.prefix) 4240 continue; 4241 4242 elf_create_prefix_symbol(file->elf, func, opts.prefix); 4243 break; 4244 } 4245 4246 if (!prev) 4247 return -1; 4248 4249 if (!insn->cfi) { 4250 /* 4251 * This can happen if stack validation isn't enabled or the 4252 * function is annotated with STACK_FRAME_NON_STANDARD. 4253 */ 4254 return 0; 4255 } 4256 4257 /* Propagate insn->cfi to the prefix code */ 4258 cfi = cfi_hash_find_or_add(insn->cfi); 4259 for (; prev != insn; prev = next_insn_same_sec(file, prev)) 4260 prev->cfi = cfi; 4261 4262 return 0; 4263 } 4264 4265 static int add_prefix_symbols(struct objtool_file *file) 4266 { 4267 struct section *sec; 4268 struct symbol *func; 4269 4270 for_each_sec(file, sec) { 4271 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 4272 continue; 4273 4274 sec_for_each_sym(sec, func) { 4275 if (func->type != STT_FUNC) 4276 continue; 4277 4278 add_prefix_symbol(file, func); 4279 } 4280 } 4281 4282 return 0; 4283 } 4284 4285 static int validate_symbol(struct objtool_file *file, struct section *sec, 4286 struct symbol *sym, struct insn_state *state) 4287 { 4288 struct instruction *insn; 4289 int ret; 4290 4291 if (!sym->len) { 4292 WARN("%s() is missing an ELF size annotation", sym->name); 4293 return 1; 4294 } 4295 4296 if (sym->pfunc != sym || sym->alias != sym) 4297 return 0; 4298 4299 insn = find_insn(file, sec, sym->offset); 4300 if (!insn || insn->ignore || insn->visited) 4301 return 0; 4302 4303 state->uaccess = sym->uaccess_safe; 4304 4305 ret = validate_branch(file, insn_func(insn), insn, *state); 4306 if (ret) 4307 BT_INSN(insn, "<=== (sym)"); 4308 return ret; 4309 } 4310 4311 static int validate_section(struct objtool_file *file, struct section *sec) 4312 { 4313 struct insn_state state; 4314 struct symbol *func; 4315 int warnings = 0; 4316 4317 sec_for_each_sym(sec, func) { 4318 if (func->type != STT_FUNC) 4319 continue; 4320 4321 init_insn_state(file, &state, sec); 4322 set_func_state(&state.cfi); 4323 4324 warnings += validate_symbol(file, sec, func, &state); 4325 } 4326 4327 return warnings; 4328 } 4329 4330 static int validate_noinstr_sections(struct objtool_file *file) 4331 { 4332 struct section *sec; 4333 int warnings = 0; 4334 4335 sec = find_section_by_name(file->elf, ".noinstr.text"); 4336 if (sec) { 4337 warnings += validate_section(file, sec); 4338 warnings += validate_unwind_hints(file, sec); 4339 } 4340 4341 sec = find_section_by_name(file->elf, ".entry.text"); 4342 if (sec) { 4343 warnings += validate_section(file, sec); 4344 warnings += validate_unwind_hints(file, sec); 4345 } 4346 4347 sec = find_section_by_name(file->elf, ".cpuidle.text"); 4348 if (sec) { 4349 warnings += validate_section(file, sec); 4350 warnings += validate_unwind_hints(file, sec); 4351 } 4352 4353 return warnings; 4354 } 4355 4356 static int validate_functions(struct objtool_file *file) 4357 { 4358 struct section *sec; 4359 int warnings = 0; 4360 4361 for_each_sec(file, sec) { 4362 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 4363 continue; 4364 4365 warnings += validate_section(file, sec); 4366 } 4367 4368 return warnings; 4369 } 4370 4371 static void mark_endbr_used(struct instruction *insn) 4372 { 4373 if (!list_empty(&insn->call_node)) 4374 list_del_init(&insn->call_node); 4375 } 4376 4377 static bool noendbr_range(struct objtool_file *file, struct instruction *insn) 4378 { 4379 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1); 4380 struct instruction *first; 4381 4382 if (!sym) 4383 return false; 4384 4385 first = find_insn(file, sym->sec, sym->offset); 4386 if (!first) 4387 return false; 4388 4389 if (first->type != INSN_ENDBR && !first->noendbr) 4390 return false; 4391 4392 return insn->offset == sym->offset + sym->len; 4393 } 4394 4395 static int __validate_ibt_insn(struct objtool_file *file, struct instruction *insn, 4396 struct instruction *dest) 4397 { 4398 if (dest->type == INSN_ENDBR) { 4399 mark_endbr_used(dest); 4400 return 0; 4401 } 4402 4403 if (insn_func(dest) && insn_func(insn) && 4404 insn_func(dest)->pfunc == insn_func(insn)->pfunc) { 4405 /* 4406 * Anything from->to self is either _THIS_IP_ or 4407 * IRET-to-self. 4408 * 4409 * There is no sane way to annotate _THIS_IP_ since the 4410 * compiler treats the relocation as a constant and is 4411 * happy to fold in offsets, skewing any annotation we 4412 * do, leading to vast amounts of false-positives. 4413 * 4414 * There's also compiler generated _THIS_IP_ through 4415 * KCOV and such which we have no hope of annotating. 4416 * 4417 * As such, blanket accept self-references without 4418 * issue. 4419 */ 4420 return 0; 4421 } 4422 4423 /* 4424 * Accept anything ANNOTATE_NOENDBR. 4425 */ 4426 if (dest->noendbr) 4427 return 0; 4428 4429 /* 4430 * Accept if this is the instruction after a symbol 4431 * that is (no)endbr -- typical code-range usage. 4432 */ 4433 if (noendbr_range(file, dest)) 4434 return 0; 4435 4436 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4437 return 1; 4438 } 4439 4440 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn) 4441 { 4442 struct instruction *dest; 4443 struct reloc *reloc; 4444 unsigned long off; 4445 int warnings = 0; 4446 4447 /* 4448 * Looking for function pointer load relocations. Ignore 4449 * direct/indirect branches: 4450 */ 4451 switch (insn->type) { 4452 4453 case INSN_CALL: 4454 case INSN_CALL_DYNAMIC: 4455 case INSN_JUMP_CONDITIONAL: 4456 case INSN_JUMP_UNCONDITIONAL: 4457 case INSN_JUMP_DYNAMIC: 4458 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4459 case INSN_RETURN: 4460 case INSN_NOP: 4461 return 0; 4462 4463 case INSN_LEA_RIP: 4464 if (!insn_reloc(file, insn)) { 4465 /* local function pointer reference without reloc */ 4466 4467 off = arch_jump_destination(insn); 4468 4469 dest = find_insn(file, insn->sec, off); 4470 if (!dest) { 4471 WARN_INSN(insn, "corrupt function pointer reference"); 4472 return 1; 4473 } 4474 4475 return __validate_ibt_insn(file, insn, dest); 4476 } 4477 break; 4478 4479 default: 4480 break; 4481 } 4482 4483 for (reloc = insn_reloc(file, insn); 4484 reloc; 4485 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 4486 reloc_offset(reloc) + 1, 4487 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) { 4488 4489 off = reloc->sym->offset; 4490 if (reloc_type(reloc) == R_X86_64_PC32 || 4491 reloc_type(reloc) == R_X86_64_PLT32) 4492 off += arch_dest_reloc_offset(reloc_addend(reloc)); 4493 else 4494 off += reloc_addend(reloc); 4495 4496 dest = find_insn(file, reloc->sym->sec, off); 4497 if (!dest) 4498 continue; 4499 4500 warnings += __validate_ibt_insn(file, insn, dest); 4501 } 4502 4503 return warnings; 4504 } 4505 4506 static int validate_ibt_data_reloc(struct objtool_file *file, 4507 struct reloc *reloc) 4508 { 4509 struct instruction *dest; 4510 4511 dest = find_insn(file, reloc->sym->sec, 4512 reloc->sym->offset + reloc_addend(reloc)); 4513 if (!dest) 4514 return 0; 4515 4516 if (dest->type == INSN_ENDBR) { 4517 mark_endbr_used(dest); 4518 return 0; 4519 } 4520 4521 if (dest->noendbr) 4522 return 0; 4523 4524 WARN_FUNC("data relocation to !ENDBR: %s", 4525 reloc->sec->base, reloc_offset(reloc), 4526 offstr(dest->sec, dest->offset)); 4527 4528 return 1; 4529 } 4530 4531 /* 4532 * Validate IBT rules and remove used ENDBR instructions from the seal list. 4533 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with 4534 * NOPs) later, in create_ibt_endbr_seal_sections(). 4535 */ 4536 static int validate_ibt(struct objtool_file *file) 4537 { 4538 struct section *sec; 4539 struct reloc *reloc; 4540 struct instruction *insn; 4541 int warnings = 0; 4542 4543 for_each_insn(file, insn) 4544 warnings += validate_ibt_insn(file, insn); 4545 4546 for_each_sec(file, sec) { 4547 4548 /* Already done by validate_ibt_insn() */ 4549 if (sec->sh.sh_flags & SHF_EXECINSTR) 4550 continue; 4551 4552 if (!sec->rsec) 4553 continue; 4554 4555 /* 4556 * These sections can reference text addresses, but not with 4557 * the intent to indirect branch to them. 4558 */ 4559 if ((!strncmp(sec->name, ".discard", 8) && 4560 strcmp(sec->name, ".discard.ibt_endbr_noseal")) || 4561 !strncmp(sec->name, ".debug", 6) || 4562 !strcmp(sec->name, ".altinstructions") || 4563 !strcmp(sec->name, ".ibt_endbr_seal") || 4564 !strcmp(sec->name, ".orc_unwind_ip") || 4565 !strcmp(sec->name, ".parainstructions") || 4566 !strcmp(sec->name, ".retpoline_sites") || 4567 !strcmp(sec->name, ".smp_locks") || 4568 !strcmp(sec->name, ".static_call_sites") || 4569 !strcmp(sec->name, "_error_injection_whitelist") || 4570 !strcmp(sec->name, "_kprobe_blacklist") || 4571 !strcmp(sec->name, "__bug_table") || 4572 !strcmp(sec->name, "__ex_table") || 4573 !strcmp(sec->name, "__jump_table") || 4574 !strcmp(sec->name, "__mcount_loc") || 4575 !strcmp(sec->name, ".kcfi_traps") || 4576 !strcmp(sec->name, "__tracepoints") || 4577 strstr(sec->name, "__patchable_function_entries")) 4578 continue; 4579 4580 for_each_reloc(sec->rsec, reloc) 4581 warnings += validate_ibt_data_reloc(file, reloc); 4582 } 4583 4584 return warnings; 4585 } 4586 4587 static int validate_sls(struct objtool_file *file) 4588 { 4589 struct instruction *insn, *next_insn; 4590 int warnings = 0; 4591 4592 for_each_insn(file, insn) { 4593 next_insn = next_insn_same_sec(file, insn); 4594 4595 if (insn->retpoline_safe) 4596 continue; 4597 4598 switch (insn->type) { 4599 case INSN_RETURN: 4600 if (!next_insn || next_insn->type != INSN_TRAP) { 4601 WARN_INSN(insn, "missing int3 after ret"); 4602 warnings++; 4603 } 4604 4605 break; 4606 case INSN_JUMP_DYNAMIC: 4607 if (!next_insn || next_insn->type != INSN_TRAP) { 4608 WARN_INSN(insn, "missing int3 after indirect jump"); 4609 warnings++; 4610 } 4611 break; 4612 default: 4613 break; 4614 } 4615 } 4616 4617 return warnings; 4618 } 4619 4620 static bool ignore_noreturn_call(struct instruction *insn) 4621 { 4622 struct symbol *call_dest = insn_call_dest(insn); 4623 4624 /* 4625 * FIXME: hack, we need a real noreturn solution 4626 * 4627 * Problem is, exc_double_fault() may or may not return, depending on 4628 * whether CONFIG_X86_ESPFIX64 is set. But objtool has no visibility 4629 * to the kernel config. 4630 * 4631 * Other potential ways to fix it: 4632 * 4633 * - have compiler communicate __noreturn functions somehow 4634 * - remove CONFIG_X86_ESPFIX64 4635 * - read the .config file 4636 * - add a cmdline option 4637 * - create a generic objtool annotation format (vs a bunch of custom 4638 * formats) and annotate it 4639 */ 4640 if (!strcmp(call_dest->name, "exc_double_fault")) { 4641 /* prevent further unreachable warnings for the caller */ 4642 insn->sym->warned = 1; 4643 return true; 4644 } 4645 4646 return false; 4647 } 4648 4649 static int validate_reachable_instructions(struct objtool_file *file) 4650 { 4651 struct instruction *insn, *prev_insn; 4652 struct symbol *call_dest; 4653 int warnings = 0; 4654 4655 if (file->ignore_unreachables) 4656 return 0; 4657 4658 for_each_insn(file, insn) { 4659 if (insn->visited || ignore_unreachable_insn(file, insn)) 4660 continue; 4661 4662 prev_insn = prev_insn_same_sec(file, insn); 4663 if (prev_insn && prev_insn->dead_end) { 4664 call_dest = insn_call_dest(prev_insn); 4665 if (call_dest && !ignore_noreturn_call(prev_insn)) { 4666 WARN_INSN(insn, "%s() is missing a __noreturn annotation", 4667 call_dest->name); 4668 warnings++; 4669 continue; 4670 } 4671 } 4672 4673 WARN_INSN(insn, "unreachable instruction"); 4674 warnings++; 4675 } 4676 4677 return warnings; 4678 } 4679 4680 /* 'funcs' is a space-separated list of function names */ 4681 static int disas_funcs(const char *funcs) 4682 { 4683 const char *objdump_str, *cross_compile; 4684 int size, ret; 4685 char *cmd; 4686 4687 cross_compile = getenv("CROSS_COMPILE"); 4688 4689 objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '" 4690 "BEGIN { split(_funcs, funcs); }" 4691 "/^$/ { func_match = 0; }" 4692 "/<.*>:/ { " 4693 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);" 4694 "for (i in funcs) {" 4695 "if (funcs[i] == f) {" 4696 "func_match = 1;" 4697 "base = strtonum(\"0x\" $1);" 4698 "break;" 4699 "}" 4700 "}" 4701 "}" 4702 "{" 4703 "if (func_match) {" 4704 "addr = strtonum(\"0x\" $1);" 4705 "printf(\"%%04x \", addr - base);" 4706 "print;" 4707 "}" 4708 "}' 1>&2"; 4709 4710 /* fake snprintf() to calculate the size */ 4711 size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1; 4712 if (size <= 0) { 4713 WARN("objdump string size calculation failed"); 4714 return -1; 4715 } 4716 4717 cmd = malloc(size); 4718 4719 /* real snprintf() */ 4720 snprintf(cmd, size, objdump_str, cross_compile, objname, funcs); 4721 ret = system(cmd); 4722 if (ret) { 4723 WARN("disassembly failed: %d", ret); 4724 return -1; 4725 } 4726 4727 return 0; 4728 } 4729 4730 static int disas_warned_funcs(struct objtool_file *file) 4731 { 4732 struct symbol *sym; 4733 char *funcs = NULL, *tmp; 4734 4735 for_each_sym(file, sym) { 4736 if (sym->warned) { 4737 if (!funcs) { 4738 funcs = malloc(strlen(sym->name) + 1); 4739 strcpy(funcs, sym->name); 4740 } else { 4741 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2); 4742 sprintf(tmp, "%s %s", funcs, sym->name); 4743 free(funcs); 4744 funcs = tmp; 4745 } 4746 } 4747 } 4748 4749 if (funcs) 4750 disas_funcs(funcs); 4751 4752 return 0; 4753 } 4754 4755 struct insn_chunk { 4756 void *addr; 4757 struct insn_chunk *next; 4758 }; 4759 4760 /* 4761 * Reduce peak RSS usage by freeing insns memory before writing the ELF file, 4762 * which can trigger more allocations for .debug_* sections whose data hasn't 4763 * been read yet. 4764 */ 4765 static void free_insns(struct objtool_file *file) 4766 { 4767 struct instruction *insn; 4768 struct insn_chunk *chunks = NULL, *chunk; 4769 4770 for_each_insn(file, insn) { 4771 if (!insn->idx) { 4772 chunk = malloc(sizeof(*chunk)); 4773 chunk->addr = insn; 4774 chunk->next = chunks; 4775 chunks = chunk; 4776 } 4777 } 4778 4779 for (chunk = chunks; chunk; chunk = chunk->next) 4780 free(chunk->addr); 4781 } 4782 4783 int check(struct objtool_file *file) 4784 { 4785 int ret, warnings = 0; 4786 4787 arch_initial_func_cfi_state(&initial_func_cfi); 4788 init_cfi_state(&init_cfi); 4789 init_cfi_state(&func_cfi); 4790 set_func_state(&func_cfi); 4791 init_cfi_state(&force_undefined_cfi); 4792 force_undefined_cfi.force_undefined = true; 4793 4794 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) 4795 goto out; 4796 4797 cfi_hash_add(&init_cfi); 4798 cfi_hash_add(&func_cfi); 4799 4800 ret = decode_sections(file); 4801 if (ret < 0) 4802 goto out; 4803 4804 warnings += ret; 4805 4806 if (!nr_insns) 4807 goto out; 4808 4809 if (opts.retpoline) { 4810 ret = validate_retpoline(file); 4811 if (ret < 0) 4812 return ret; 4813 warnings += ret; 4814 } 4815 4816 if (opts.stackval || opts.orc || opts.uaccess) { 4817 ret = validate_functions(file); 4818 if (ret < 0) 4819 goto out; 4820 warnings += ret; 4821 4822 ret = validate_unwind_hints(file, NULL); 4823 if (ret < 0) 4824 goto out; 4825 warnings += ret; 4826 4827 if (!warnings) { 4828 ret = validate_reachable_instructions(file); 4829 if (ret < 0) 4830 goto out; 4831 warnings += ret; 4832 } 4833 4834 } else if (opts.noinstr) { 4835 ret = validate_noinstr_sections(file); 4836 if (ret < 0) 4837 goto out; 4838 warnings += ret; 4839 } 4840 4841 if (opts.unret) { 4842 /* 4843 * Must be after validate_branch() and friends, it plays 4844 * further games with insn->visited. 4845 */ 4846 ret = validate_unrets(file); 4847 if (ret < 0) 4848 return ret; 4849 warnings += ret; 4850 } 4851 4852 if (opts.ibt) { 4853 ret = validate_ibt(file); 4854 if (ret < 0) 4855 goto out; 4856 warnings += ret; 4857 } 4858 4859 if (opts.sls) { 4860 ret = validate_sls(file); 4861 if (ret < 0) 4862 goto out; 4863 warnings += ret; 4864 } 4865 4866 if (opts.static_call) { 4867 ret = create_static_call_sections(file); 4868 if (ret < 0) 4869 goto out; 4870 warnings += ret; 4871 } 4872 4873 if (opts.retpoline) { 4874 ret = create_retpoline_sites_sections(file); 4875 if (ret < 0) 4876 goto out; 4877 warnings += ret; 4878 } 4879 4880 if (opts.cfi) { 4881 ret = create_cfi_sections(file); 4882 if (ret < 0) 4883 goto out; 4884 warnings += ret; 4885 } 4886 4887 if (opts.rethunk) { 4888 ret = create_return_sites_sections(file); 4889 if (ret < 0) 4890 goto out; 4891 warnings += ret; 4892 4893 if (opts.hack_skylake) { 4894 ret = create_direct_call_sections(file); 4895 if (ret < 0) 4896 goto out; 4897 warnings += ret; 4898 } 4899 } 4900 4901 if (opts.mcount) { 4902 ret = create_mcount_loc_sections(file); 4903 if (ret < 0) 4904 goto out; 4905 warnings += ret; 4906 } 4907 4908 if (opts.prefix) { 4909 ret = add_prefix_symbols(file); 4910 if (ret < 0) 4911 return ret; 4912 warnings += ret; 4913 } 4914 4915 if (opts.ibt) { 4916 ret = create_ibt_endbr_seal_sections(file); 4917 if (ret < 0) 4918 goto out; 4919 warnings += ret; 4920 } 4921 4922 if (opts.orc && nr_insns) { 4923 ret = orc_create(file); 4924 if (ret < 0) 4925 goto out; 4926 warnings += ret; 4927 } 4928 4929 free_insns(file); 4930 4931 if (opts.verbose) 4932 disas_warned_funcs(file); 4933 4934 if (opts.stats) { 4935 printf("nr_insns_visited: %ld\n", nr_insns_visited); 4936 printf("nr_cfi: %ld\n", nr_cfi); 4937 printf("nr_cfi_reused: %ld\n", nr_cfi_reused); 4938 printf("nr_cfi_cache: %ld\n", nr_cfi_cache); 4939 } 4940 4941 out: 4942 /* 4943 * For now, don't fail the kernel build on fatal warnings. These 4944 * errors are still fairly common due to the growing matrix of 4945 * supported toolchains and their recent pace of change. 4946 */ 4947 return 0; 4948 } 4949