1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6 #define _GNU_SOURCE /* memmem() */ 7 #include <fnmatch.h> 8 #include <string.h> 9 #include <stdlib.h> 10 #include <inttypes.h> 11 #include <sys/mman.h> 12 13 #include <objtool/builtin.h> 14 #include <objtool/cfi.h> 15 #include <objtool/arch.h> 16 #include <objtool/disas.h> 17 #include <objtool/check.h> 18 #include <objtool/special.h> 19 #include <objtool/trace.h> 20 #include <objtool/warn.h> 21 #include <objtool/checksum.h> 22 #include <objtool/util.h> 23 24 #include <linux/objtool_types.h> 25 #include <linux/hashtable.h> 26 #include <linux/kernel.h> 27 #include <linux/static_call_types.h> 28 #include <linux/string.h> 29 30 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 31 32 static struct cfi_init_state initial_func_cfi; 33 static struct cfi_state init_cfi; 34 static struct cfi_state func_cfi; 35 static struct cfi_state force_undefined_cfi; 36 37 struct disas_context *objtool_disas_ctx; 38 39 size_t sym_name_max_len; 40 41 struct instruction *find_insn(struct objtool_file *file, 42 struct section *sec, unsigned long offset) 43 { 44 struct instruction *insn; 45 46 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) { 47 if (insn->sec == sec && insn->offset == offset) 48 return insn; 49 } 50 51 return NULL; 52 } 53 54 struct instruction *next_insn_same_sec(struct objtool_file *file, 55 struct instruction *insn) 56 { 57 if (insn->idx == INSN_CHUNK_MAX) 58 return find_insn(file, insn->sec, insn->offset + insn->len); 59 60 insn++; 61 if (!insn->len) 62 return NULL; 63 64 return insn; 65 } 66 67 static struct instruction *next_insn_same_func(struct objtool_file *file, 68 struct instruction *insn) 69 { 70 struct instruction *next = next_insn_same_sec(file, insn); 71 struct symbol *func = insn_func(insn); 72 73 if (!func) 74 return NULL; 75 76 if (next && insn_func(next) == func) 77 return next; 78 79 /* Check if we're already in the subfunction: */ 80 if (func == func->cfunc) 81 return NULL; 82 83 /* Move to the subfunction: */ 84 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 85 } 86 87 static struct instruction *prev_insn_same_sec(struct objtool_file *file, 88 struct instruction *insn) 89 { 90 if (insn->idx == 0) { 91 if (insn->prev_len) 92 return find_insn(file, insn->sec, insn->offset - insn->prev_len); 93 return NULL; 94 } 95 96 return insn - 1; 97 } 98 99 static struct instruction *prev_insn_same_sym(struct objtool_file *file, 100 struct instruction *insn) 101 { 102 struct instruction *prev = prev_insn_same_sec(file, insn); 103 104 if (prev && insn_func(prev) == insn_func(insn)) 105 return prev; 106 107 return NULL; 108 } 109 110 #define for_each_insn(file, insn) \ 111 for (struct section *__sec, *__fake = (struct section *)1; \ 112 __fake; __fake = NULL) \ 113 for_each_sec(file->elf, __sec) \ 114 sec_for_each_insn(file, __sec, insn) 115 116 #define func_for_each_insn(file, func, insn) \ 117 for (insn = find_insn(file, func->sec, func->offset); \ 118 insn; \ 119 insn = next_insn_same_func(file, insn)) 120 121 #define sym_for_each_insn(file, sym, insn) \ 122 for (insn = find_insn(file, sym->sec, sym->offset); \ 123 insn && insn->offset < sym->offset + sym->len; \ 124 insn = next_insn_same_sec(file, insn)) 125 126 #define sym_for_each_insn_continue_reverse(file, sym, insn) \ 127 for (insn = prev_insn_same_sec(file, insn); \ 128 insn && insn->offset >= sym->offset; \ 129 insn = prev_insn_same_sec(file, insn)) 130 131 #define sec_for_each_insn_from(file, insn) \ 132 for (; insn; insn = next_insn_same_sec(file, insn)) 133 134 #define sec_for_each_insn_continue(file, insn) \ 135 for (insn = next_insn_same_sec(file, insn); insn; \ 136 insn = next_insn_same_sec(file, insn)) 137 138 static inline struct reloc *insn_jump_table(struct instruction *insn) 139 { 140 if (insn->type == INSN_JUMP_DYNAMIC || 141 insn->type == INSN_CALL_DYNAMIC) 142 return insn->_jump_table; 143 144 return NULL; 145 } 146 147 static inline unsigned long insn_jump_table_size(struct instruction *insn) 148 { 149 if (insn->type == INSN_JUMP_DYNAMIC || 150 insn->type == INSN_CALL_DYNAMIC) 151 return insn->_jump_table_size; 152 153 return 0; 154 } 155 156 static bool is_jump_table_jump(struct instruction *insn) 157 { 158 struct alt_group *alt_group = insn->alt_group; 159 160 if (insn_jump_table(insn)) 161 return true; 162 163 /* Retpoline alternative for a jump table? */ 164 return alt_group && alt_group->orig_group && 165 insn_jump_table(alt_group->orig_group->first_insn); 166 } 167 168 static bool is_sibling_call(struct instruction *insn) 169 { 170 /* 171 * Assume only STT_FUNC calls have jump-tables. 172 */ 173 if (insn_func(insn)) { 174 /* An indirect jump is either a sibling call or a jump to a table. */ 175 if (insn->type == INSN_JUMP_DYNAMIC) 176 return !is_jump_table_jump(insn); 177 } 178 179 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */ 180 return (is_static_jump(insn) && insn_call_dest(insn)); 181 } 182 183 /* 184 * Checks if a function is a Rust "noreturn" one. 185 */ 186 static bool is_rust_noreturn(const struct symbol *func) 187 { 188 /* 189 * If it does not start with "_R", then it is not a Rust symbol. 190 */ 191 if (strncmp(func->name, "_R", 2)) 192 return false; 193 194 /* 195 * These are just heuristics -- we do not control the precise symbol 196 * name, due to the crate disambiguators (which depend on the compiler) 197 * as well as changes to the source code itself between versions (since 198 * these come from the Rust standard library). 199 */ 200 return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || 201 str_ends_with(func->name, "_4core6option13expect_failed") || 202 str_ends_with(func->name, "_4core6option13unwrap_failed") || 203 str_ends_with(func->name, "_4core6result13unwrap_failed") || 204 str_ends_with(func->name, "_4core9panicking5panic") || 205 str_ends_with(func->name, "_4core9panicking9panic_fmt") || 206 str_ends_with(func->name, "_4core9panicking14panic_explicit") || 207 str_ends_with(func->name, "_4core9panicking14panic_nounwind") || 208 str_ends_with(func->name, "_4core9panicking18panic_bounds_check") || 209 str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt") || 210 str_ends_with(func->name, "_4core9panicking19assert_failed_inner") || 211 str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference") || 212 str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") || 213 str_ends_with(func->name, "_7___rustc17rust_begin_unwind") || 214 strstr(func->name, "_4core9panicking13assert_failed") || 215 strstr(func->name, "_4core9panicking11panic_const24panic_const_") || 216 (strstr(func->name, "_4core5slice5index") && 217 strstr(func->name, "slice_") && 218 str_ends_with(func->name, "_fail")); 219 } 220 221 /* 222 * This checks to see if the given function is a "noreturn" function. 223 * 224 * For global functions which are outside the scope of this object file, we 225 * have to keep a manual list of them. 226 * 227 * For local functions, we have to detect them manually by simply looking for 228 * the lack of a return instruction. 229 */ 230 static bool __dead_end_function(struct objtool_file *file, struct symbol *func, 231 int recursion) 232 { 233 int i; 234 struct instruction *insn; 235 bool empty = true; 236 237 #define NORETURN(func) __stringify(func), 238 static const char * const global_noreturns[] = { 239 #include "noreturns.h" 240 }; 241 #undef NORETURN 242 243 if (!func) 244 return false; 245 246 if (!is_local_sym(func)) { 247 if (is_rust_noreturn(func)) 248 return true; 249 250 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) 251 if (!strcmp(func->name, global_noreturns[i])) 252 return true; 253 } 254 255 if (is_weak_sym(func)) 256 return false; 257 258 if (!func->len) 259 return false; 260 261 insn = find_insn(file, func->sec, func->offset); 262 if (!insn || !insn_func(insn)) 263 return false; 264 265 func_for_each_insn(file, func, insn) { 266 empty = false; 267 268 if (insn->type == INSN_RETURN) 269 return false; 270 } 271 272 if (empty) 273 return false; 274 275 /* 276 * A function can have a sibling call instead of a return. In that 277 * case, the function's dead-end status depends on whether the target 278 * of the sibling call returns. 279 */ 280 func_for_each_insn(file, func, insn) { 281 if (is_sibling_call(insn)) { 282 struct instruction *dest = insn->jump_dest; 283 284 if (!dest) 285 /* sibling call to another file */ 286 return false; 287 288 /* local sibling call */ 289 if (recursion == 5) { 290 /* 291 * Infinite recursion: two functions have 292 * sibling calls to each other. This is a very 293 * rare case. It means they aren't dead ends. 294 */ 295 return false; 296 } 297 298 return __dead_end_function(file, insn_func(dest), recursion+1); 299 } 300 } 301 302 return true; 303 } 304 305 static bool dead_end_function(struct objtool_file *file, struct symbol *func) 306 { 307 return __dead_end_function(file, func, 0); 308 } 309 310 static void init_cfi_state(struct cfi_state *cfi) 311 { 312 int i; 313 314 for (i = 0; i < CFI_NUM_REGS; i++) { 315 cfi->regs[i].base = CFI_UNDEFINED; 316 cfi->vals[i].base = CFI_UNDEFINED; 317 } 318 cfi->cfa.base = CFI_UNDEFINED; 319 cfi->drap_reg = CFI_UNDEFINED; 320 cfi->drap_offset = -1; 321 } 322 323 static void init_insn_state(struct objtool_file *file, struct insn_state *state, 324 struct section *sec) 325 { 326 memset(state, 0, sizeof(*state)); 327 init_cfi_state(&state->cfi); 328 329 if (opts.noinstr && sec) 330 state->noinstr = sec->noinstr; 331 } 332 333 static struct cfi_state *cfi_alloc(void) 334 { 335 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state)); 336 if (!cfi) { 337 ERROR_GLIBC("calloc"); 338 exit(1); 339 } 340 nr_cfi++; 341 return cfi; 342 } 343 344 static int cfi_bits; 345 static struct hlist_head *cfi_hash; 346 347 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2) 348 { 349 return memcmp((void *)cfi1 + sizeof(cfi1->hash), 350 (void *)cfi2 + sizeof(cfi2->hash), 351 sizeof(struct cfi_state) - sizeof(struct hlist_node)); 352 } 353 354 static inline u32 cfi_key(struct cfi_state *cfi) 355 { 356 return jhash((void *)cfi + sizeof(cfi->hash), 357 sizeof(*cfi) - sizeof(cfi->hash), 0); 358 } 359 360 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi) 361 { 362 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 363 struct cfi_state *obj; 364 365 hlist_for_each_entry(obj, head, hash) { 366 if (!cficmp(cfi, obj)) { 367 nr_cfi_cache++; 368 return obj; 369 } 370 } 371 372 obj = cfi_alloc(); 373 *obj = *cfi; 374 hlist_add_head(&obj->hash, head); 375 376 return obj; 377 } 378 379 static void cfi_hash_add(struct cfi_state *cfi) 380 { 381 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 382 383 hlist_add_head(&cfi->hash, head); 384 } 385 386 static void *cfi_hash_alloc(unsigned long size) 387 { 388 cfi_bits = max(10, ilog2(size)); 389 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits, 390 PROT_READ|PROT_WRITE, 391 MAP_PRIVATE|MAP_ANON, -1, 0); 392 if (cfi_hash == (void *)-1L) { 393 ERROR_GLIBC("mmap fail cfi_hash"); 394 cfi_hash = NULL; 395 } else if (opts.stats) { 396 printf("cfi_bits: %d\n", cfi_bits); 397 } 398 399 return cfi_hash; 400 } 401 402 static unsigned long nr_insns; 403 static unsigned long nr_insns_visited; 404 405 /* 406 * Call the arch-specific instruction decoder for all the instructions and add 407 * them to the global instruction list. 408 */ 409 static int decode_instructions(struct objtool_file *file) 410 { 411 struct section *sec; 412 struct symbol *func; 413 unsigned long offset; 414 struct instruction *insn; 415 416 for_each_sec(file->elf, sec) { 417 struct instruction *insns = NULL; 418 u8 prev_len = 0; 419 u8 idx = 0; 420 421 if (!is_text_sec(sec)) 422 continue; 423 424 if (strcmp(sec->name, ".altinstr_replacement") && 425 strcmp(sec->name, ".altinstr_aux") && 426 strncmp(sec->name, ".discard.", 9)) 427 sec->text = true; 428 429 if (!strcmp(sec->name, ".noinstr.text") || 430 !strcmp(sec->name, ".entry.text") || 431 !strcmp(sec->name, ".cpuidle.text") || 432 !strncmp(sec->name, ".text..__x86.", 13)) 433 sec->noinstr = true; 434 435 /* 436 * .init.text code is ran before userspace and thus doesn't 437 * strictly need retpolines, except for modules which are 438 * loaded late, they very much do need retpoline in their 439 * .init.text 440 */ 441 if (!strcmp(sec->name, ".init.text") && !opts.module) 442 sec->init = true; 443 444 for (offset = 0; offset < sec_size(sec); offset += insn->len) { 445 if (!insns || idx == INSN_CHUNK_MAX) { 446 insns = calloc(INSN_CHUNK_SIZE, sizeof(*insn)); 447 if (!insns) { 448 ERROR_GLIBC("calloc"); 449 return -1; 450 } 451 idx = 0; 452 } else { 453 idx++; 454 } 455 insn = &insns[idx]; 456 insn->idx = idx; 457 458 INIT_LIST_HEAD(&insn->call_node); 459 insn->sec = sec; 460 insn->offset = offset; 461 insn->prev_len = prev_len; 462 463 if (arch_decode_instruction(file, sec, offset, sec_size(sec) - offset, insn)) 464 return -1; 465 466 prev_len = insn->len; 467 468 /* 469 * By default, "ud2" is a dead end unless otherwise 470 * annotated, because GCC 7 inserts it for certain 471 * divide-by-zero cases. 472 */ 473 if (insn->type == INSN_BUG) 474 insn->dead_end = true; 475 476 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset)); 477 nr_insns++; 478 } 479 480 sec_for_each_sym(sec, func) { 481 if (!is_notype_sym(func) && !is_func_sym(func)) 482 continue; 483 484 if (func->offset == sec_size(sec)) { 485 /* Heuristic: likely an "end" symbol */ 486 if (is_notype_sym(func)) 487 continue; 488 ERROR("%s(): STT_FUNC at end of section", func->name); 489 return -1; 490 } 491 492 if (func->embedded_insn || func->alias != func) 493 continue; 494 495 if (!find_insn(file, sec, func->offset)) { 496 ERROR("%s(): can't find starting instruction", func->name); 497 return -1; 498 } 499 500 sym_for_each_insn(file, func, insn) { 501 insn->sym = func; 502 if (is_func_sym(func) && 503 insn->type == INSN_ENDBR && 504 list_empty(&insn->call_node)) { 505 if (insn->offset == func->offset) { 506 list_add_tail(&insn->call_node, &file->endbr_list); 507 file->nr_endbr++; 508 } else { 509 file->nr_endbr_int++; 510 } 511 } 512 } 513 } 514 } 515 516 if (opts.stats) 517 printf("nr_insns: %lu\n", nr_insns); 518 519 return 0; 520 } 521 522 /* 523 * Read the pv_ops[] .data table to find the static initialized values. 524 */ 525 static int add_pv_ops(struct objtool_file *file, const char *symname) 526 { 527 struct symbol *sym, *func; 528 unsigned long off, end; 529 struct reloc *reloc; 530 int idx; 531 532 sym = find_symbol_by_name(file->elf, symname); 533 if (!sym) 534 return 0; 535 536 off = sym->offset; 537 end = off + sym->len; 538 for (;;) { 539 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 540 if (!reloc) 541 break; 542 543 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long); 544 545 func = reloc->sym; 546 if (is_sec_sym(func)) 547 func = find_symbol_by_offset(reloc->sym->sec, 548 reloc_addend(reloc)); 549 if (!func) { 550 ERROR_FUNC(reloc->sym->sec, reloc_addend(reloc), 551 "can't find func at %s[%d]", symname, idx); 552 return -1; 553 } 554 555 if (objtool_pv_add(file, idx, func)) 556 return -1; 557 558 off = reloc_offset(reloc) + 1; 559 if (off > end) 560 break; 561 } 562 563 return 0; 564 } 565 566 /* 567 * Allocate and initialize file->pv_ops[]. 568 */ 569 static int init_pv_ops(struct objtool_file *file) 570 { 571 static const char *pv_ops_tables[] = { 572 "pv_ops", 573 "xen_cpu_ops", 574 "xen_irq_ops", 575 "xen_mmu_ops", 576 NULL, 577 }; 578 const char *pv_ops; 579 struct symbol *sym; 580 int idx, nr; 581 582 if (!opts.noinstr) 583 return 0; 584 585 file->pv_ops = NULL; 586 587 sym = find_symbol_by_name(file->elf, "pv_ops"); 588 if (!sym) 589 return 0; 590 591 nr = sym->len / sizeof(unsigned long); 592 file->pv_ops = calloc(nr, sizeof(struct pv_state)); 593 if (!file->pv_ops) { 594 ERROR_GLIBC("calloc"); 595 return -1; 596 } 597 598 for (idx = 0; idx < nr; idx++) 599 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 600 601 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) { 602 if (add_pv_ops(file, pv_ops)) 603 return -1; 604 } 605 606 return 0; 607 } 608 609 static bool is_livepatch_module(struct objtool_file *file) 610 { 611 struct section *sec; 612 613 if (!opts.module) 614 return false; 615 616 sec = find_section_by_name(file->elf, ".modinfo"); 617 if (!sec) 618 return false; 619 620 return memmem(sec->data->d_buf, sec_size(sec), "\0livepatch=Y", 12); 621 } 622 623 static int create_static_call_sections(struct objtool_file *file) 624 { 625 struct static_call_site *site; 626 struct section *sec; 627 struct instruction *insn; 628 struct symbol *key_sym; 629 char *key_name, *tmp; 630 int idx; 631 632 sec = find_section_by_name(file->elf, ".static_call_sites"); 633 if (sec) { 634 /* 635 * Livepatch modules may have already extracted the static call 636 * site entries to take advantage of vmlinux static call 637 * privileges. 638 */ 639 if (!file->klp) 640 WARN("file already has .static_call_sites section, skipping"); 641 642 return 0; 643 } 644 645 if (list_empty(&file->static_call_list)) 646 return 0; 647 648 idx = 0; 649 list_for_each_entry(insn, &file->static_call_list, call_node) 650 idx++; 651 652 sec = elf_create_section_pair(file->elf, ".static_call_sites", 653 sizeof(*site), idx, idx * 2); 654 if (!sec) 655 return -1; 656 657 /* Allow modules to modify the low bits of static_call_site::key */ 658 sec->sh.sh_flags |= SHF_WRITE; 659 660 idx = 0; 661 list_for_each_entry(insn, &file->static_call_list, call_node) { 662 663 /* populate reloc for 'addr' */ 664 if (!elf_init_reloc_text_sym(file->elf, sec, 665 idx * sizeof(*site), idx * 2, 666 insn->sec, insn->offset)) 667 return -1; 668 669 /* find key symbol */ 670 key_name = strdup(insn_call_dest(insn)->name); 671 if (!key_name) { 672 ERROR_GLIBC("strdup"); 673 return -1; 674 } 675 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR, 676 STATIC_CALL_TRAMP_PREFIX_LEN)) { 677 ERROR("static_call: trampoline name malformed: %s", key_name); 678 return -1; 679 } 680 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN; 681 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN); 682 683 key_sym = find_symbol_by_name(file->elf, tmp); 684 if (!key_sym) { 685 if (!opts.module || file->klp) { 686 ERROR("static_call: can't find static_call_key symbol: %s", tmp); 687 return -1; 688 } 689 690 /* 691 * For modules(), the key might not be exported, which 692 * means the module can make static calls but isn't 693 * allowed to change them. 694 * 695 * In that case we temporarily set the key to be the 696 * trampoline address. This is fixed up in 697 * static_call_add_module(). 698 */ 699 key_sym = insn_call_dest(insn); 700 } 701 702 /* populate reloc for 'key' */ 703 if (!elf_init_reloc_data_sym(file->elf, sec, 704 idx * sizeof(*site) + 4, 705 (idx * 2) + 1, key_sym, 706 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL)) 707 return -1; 708 709 idx++; 710 } 711 712 return 0; 713 } 714 715 static int create_retpoline_sites_sections(struct objtool_file *file) 716 { 717 struct instruction *insn; 718 struct section *sec; 719 int idx; 720 721 sec = find_section_by_name(file->elf, ".retpoline_sites"); 722 if (sec) { 723 WARN("file already has .retpoline_sites, skipping"); 724 return 0; 725 } 726 727 idx = 0; 728 list_for_each_entry(insn, &file->retpoline_call_list, call_node) 729 idx++; 730 731 if (!idx) 732 return 0; 733 734 sec = elf_create_section_pair(file->elf, ".retpoline_sites", 735 sizeof(int), idx, idx); 736 if (!sec) 737 return -1; 738 739 idx = 0; 740 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 741 742 if (!elf_init_reloc_text_sym(file->elf, sec, 743 idx * sizeof(int), idx, 744 insn->sec, insn->offset)) 745 return -1; 746 747 idx++; 748 } 749 750 return 0; 751 } 752 753 static int create_return_sites_sections(struct objtool_file *file) 754 { 755 struct instruction *insn; 756 struct section *sec; 757 int idx; 758 759 sec = find_section_by_name(file->elf, ".return_sites"); 760 if (sec) { 761 WARN("file already has .return_sites, skipping"); 762 return 0; 763 } 764 765 idx = 0; 766 list_for_each_entry(insn, &file->return_thunk_list, call_node) 767 idx++; 768 769 if (!idx) 770 return 0; 771 772 sec = elf_create_section_pair(file->elf, ".return_sites", 773 sizeof(int), idx, idx); 774 if (!sec) 775 return -1; 776 777 idx = 0; 778 list_for_each_entry(insn, &file->return_thunk_list, call_node) { 779 780 if (!elf_init_reloc_text_sym(file->elf, sec, 781 idx * sizeof(int), idx, 782 insn->sec, insn->offset)) 783 return -1; 784 785 idx++; 786 } 787 788 return 0; 789 } 790 791 static int create_ibt_endbr_seal_sections(struct objtool_file *file) 792 { 793 struct instruction *insn; 794 struct section *sec; 795 int idx; 796 797 sec = find_section_by_name(file->elf, ".ibt_endbr_seal"); 798 if (sec) { 799 WARN("file already has .ibt_endbr_seal, skipping"); 800 return 0; 801 } 802 803 idx = 0; 804 list_for_each_entry(insn, &file->endbr_list, call_node) 805 idx++; 806 807 if (opts.stats) { 808 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); 809 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); 810 printf("ibt: superfluous ENDBR: %d\n", idx); 811 } 812 813 if (!idx) 814 return 0; 815 816 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal", 817 sizeof(int), idx, idx); 818 if (!sec) 819 return -1; 820 821 idx = 0; 822 list_for_each_entry(insn, &file->endbr_list, call_node) { 823 824 int *site = (int *)sec->data->d_buf + idx; 825 struct symbol *sym = insn->sym; 826 *site = 0; 827 828 if (opts.module && sym && is_func_sym(sym) && 829 insn->offset == sym->offset && 830 (!strcmp(sym->name, "init_module") || 831 !strcmp(sym->name, "cleanup_module"))) { 832 ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead", 833 sym->name); 834 return -1; 835 } 836 837 if (!elf_init_reloc_text_sym(file->elf, sec, 838 idx * sizeof(int), idx, 839 insn->sec, insn->offset)) 840 return -1; 841 842 idx++; 843 } 844 845 return 0; 846 } 847 848 static int create_cfi_sections(struct objtool_file *file) 849 { 850 struct section *sec; 851 struct symbol *sym; 852 int idx; 853 854 sec = find_section_by_name(file->elf, ".cfi_sites"); 855 if (sec) { 856 WARN("file already has .cfi_sites section, skipping"); 857 return 0; 858 } 859 860 idx = 0; 861 for_each_sym(file->elf, sym) { 862 if (!is_func_sym(sym)) 863 continue; 864 865 if (strncmp(sym->name, "__cfi_", 6)) 866 continue; 867 868 idx++; 869 } 870 871 sec = elf_create_section_pair(file->elf, ".cfi_sites", 872 sizeof(unsigned int), idx, idx); 873 if (!sec) 874 return -1; 875 876 idx = 0; 877 for_each_sym(file->elf, sym) { 878 if (!is_func_sym(sym)) 879 continue; 880 881 if (strncmp(sym->name, "__cfi_", 6)) 882 continue; 883 884 if (!elf_init_reloc_text_sym(file->elf, sec, 885 idx * sizeof(unsigned int), idx, 886 sym->sec, sym->offset)) 887 return -1; 888 889 idx++; 890 } 891 892 return 0; 893 } 894 895 static int create_mcount_loc_sections(struct objtool_file *file) 896 { 897 size_t addr_size = elf_addr_size(file->elf); 898 struct instruction *insn; 899 struct section *sec; 900 int idx; 901 902 sec = find_section_by_name(file->elf, "__mcount_loc"); 903 if (sec) { 904 /* 905 * Livepatch modules have already extracted their __mcount_loc 906 * entries to cover the !CONFIG_FTRACE_MCOUNT_USE_OBJTOOL case. 907 */ 908 if (!file->klp) 909 WARN("file already has __mcount_loc section, skipping"); 910 911 return 0; 912 } 913 914 if (list_empty(&file->mcount_loc_list)) 915 return 0; 916 917 idx = 0; 918 list_for_each_entry(insn, &file->mcount_loc_list, call_node) 919 idx++; 920 921 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size, 922 idx, idx); 923 if (!sec) 924 return -1; 925 926 sec->sh.sh_addralign = addr_size; 927 928 idx = 0; 929 list_for_each_entry(insn, &file->mcount_loc_list, call_node) { 930 931 struct reloc *reloc; 932 933 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx, 934 insn->sec, insn->offset); 935 if (!reloc) 936 return -1; 937 938 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32); 939 940 idx++; 941 } 942 943 return 0; 944 } 945 946 static int create_direct_call_sections(struct objtool_file *file) 947 { 948 struct instruction *insn; 949 struct section *sec; 950 int idx; 951 952 sec = find_section_by_name(file->elf, ".call_sites"); 953 if (sec) { 954 WARN("file already has .call_sites section, skipping"); 955 return 0; 956 } 957 958 if (list_empty(&file->call_list)) 959 return 0; 960 961 idx = 0; 962 list_for_each_entry(insn, &file->call_list, call_node) 963 idx++; 964 965 sec = elf_create_section_pair(file->elf, ".call_sites", 966 sizeof(unsigned int), idx, idx); 967 if (!sec) 968 return -1; 969 970 idx = 0; 971 list_for_each_entry(insn, &file->call_list, call_node) { 972 973 if (!elf_init_reloc_text_sym(file->elf, sec, 974 idx * sizeof(unsigned int), idx, 975 insn->sec, insn->offset)) 976 return -1; 977 978 idx++; 979 } 980 981 return 0; 982 } 983 984 #ifdef BUILD_KLP 985 static int create_sym_checksum_section(struct objtool_file *file) 986 { 987 struct section *sec; 988 struct symbol *sym; 989 unsigned int idx = 0; 990 struct sym_checksum *checksum; 991 size_t entsize = sizeof(struct sym_checksum); 992 993 sec = find_section_by_name(file->elf, ".discard.sym_checksum"); 994 if (sec) { 995 if (!opts.dryrun) 996 WARN("file already has .discard.sym_checksum section, skipping"); 997 998 return 0; 999 } 1000 1001 for_each_sym(file->elf, sym) 1002 if (sym->csum.checksum) 1003 idx++; 1004 1005 if (!idx) 1006 return 0; 1007 1008 sec = elf_create_section_pair(file->elf, ".discard.sym_checksum", entsize, 1009 idx, idx); 1010 if (!sec) 1011 return -1; 1012 1013 idx = 0; 1014 for_each_sym(file->elf, sym) { 1015 if (!sym->csum.checksum) 1016 continue; 1017 1018 if (!elf_init_reloc(file->elf, sec->rsec, idx, idx * entsize, 1019 sym, 0, R_TEXT64)) 1020 return -1; 1021 1022 checksum = (struct sym_checksum *)sec->data->d_buf + idx; 1023 checksum->addr = 0; /* reloc */ 1024 checksum->checksum = sym->csum.checksum; 1025 1026 mark_sec_changed(file->elf, sec, true); 1027 1028 idx++; 1029 } 1030 1031 return 0; 1032 } 1033 #else 1034 static int create_sym_checksum_section(struct objtool_file *file) { return -EINVAL; } 1035 #endif 1036 1037 /* 1038 * Warnings shouldn't be reported for ignored functions. 1039 */ 1040 static int add_ignores(struct objtool_file *file) 1041 { 1042 struct section *rsec; 1043 struct symbol *func; 1044 struct reloc *reloc; 1045 1046 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard"); 1047 if (!rsec) 1048 return 0; 1049 1050 for_each_reloc(rsec, reloc) { 1051 switch (reloc->sym->type) { 1052 case STT_FUNC: 1053 func = reloc->sym; 1054 break; 1055 1056 case STT_SECTION: 1057 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc)); 1058 if (!func) 1059 continue; 1060 break; 1061 1062 default: 1063 ERROR("unexpected relocation symbol type in %s: %d", 1064 rsec->name, reloc->sym->type); 1065 return -1; 1066 } 1067 1068 func->ignore = true; 1069 if (func->cfunc) 1070 func->cfunc->ignore = true; 1071 } 1072 1073 return 0; 1074 } 1075 1076 /* 1077 * This is a whitelist of functions that is allowed to be called with AC set. 1078 * The list is meant to be minimal and only contains compiler instrumentation 1079 * ABI and a few functions used to implement *_{to,from}_user() functions. 1080 * 1081 * These functions must not directly change AC, but may PUSHF/POPF. 1082 */ 1083 static const char *uaccess_safe_builtin[] = { 1084 /* KASAN */ 1085 "kasan_report", 1086 "kasan_check_range", 1087 /* KASAN out-of-line */ 1088 "__asan_loadN_noabort", 1089 "__asan_load1_noabort", 1090 "__asan_load2_noabort", 1091 "__asan_load4_noabort", 1092 "__asan_load8_noabort", 1093 "__asan_load16_noabort", 1094 "__asan_storeN_noabort", 1095 "__asan_store1_noabort", 1096 "__asan_store2_noabort", 1097 "__asan_store4_noabort", 1098 "__asan_store8_noabort", 1099 "__asan_store16_noabort", 1100 "__kasan_check_read", 1101 "__kasan_check_write", 1102 /* KASAN in-line */ 1103 "__asan_report_load_n_noabort", 1104 "__asan_report_load1_noabort", 1105 "__asan_report_load2_noabort", 1106 "__asan_report_load4_noabort", 1107 "__asan_report_load8_noabort", 1108 "__asan_report_load16_noabort", 1109 "__asan_report_store_n_noabort", 1110 "__asan_report_store1_noabort", 1111 "__asan_report_store2_noabort", 1112 "__asan_report_store4_noabort", 1113 "__asan_report_store8_noabort", 1114 "__asan_report_store16_noabort", 1115 /* KCSAN */ 1116 "__kcsan_check_access", 1117 "__kcsan_mb", 1118 "__kcsan_wmb", 1119 "__kcsan_rmb", 1120 "__kcsan_release", 1121 "kcsan_found_watchpoint", 1122 "kcsan_setup_watchpoint", 1123 "kcsan_check_scoped_accesses", 1124 "kcsan_disable_current", 1125 "kcsan_enable_current_nowarn", 1126 /* KCSAN/TSAN */ 1127 "__tsan_func_entry", 1128 "__tsan_func_exit", 1129 "__tsan_read_range", 1130 "__tsan_write_range", 1131 "__tsan_read1", 1132 "__tsan_read2", 1133 "__tsan_read4", 1134 "__tsan_read8", 1135 "__tsan_read16", 1136 "__tsan_write1", 1137 "__tsan_write2", 1138 "__tsan_write4", 1139 "__tsan_write8", 1140 "__tsan_write16", 1141 "__tsan_read_write1", 1142 "__tsan_read_write2", 1143 "__tsan_read_write4", 1144 "__tsan_read_write8", 1145 "__tsan_read_write16", 1146 "__tsan_volatile_read1", 1147 "__tsan_volatile_read2", 1148 "__tsan_volatile_read4", 1149 "__tsan_volatile_read8", 1150 "__tsan_volatile_read16", 1151 "__tsan_volatile_write1", 1152 "__tsan_volatile_write2", 1153 "__tsan_volatile_write4", 1154 "__tsan_volatile_write8", 1155 "__tsan_volatile_write16", 1156 "__tsan_atomic8_load", 1157 "__tsan_atomic16_load", 1158 "__tsan_atomic32_load", 1159 "__tsan_atomic64_load", 1160 "__tsan_atomic8_store", 1161 "__tsan_atomic16_store", 1162 "__tsan_atomic32_store", 1163 "__tsan_atomic64_store", 1164 "__tsan_atomic8_exchange", 1165 "__tsan_atomic16_exchange", 1166 "__tsan_atomic32_exchange", 1167 "__tsan_atomic64_exchange", 1168 "__tsan_atomic8_fetch_add", 1169 "__tsan_atomic16_fetch_add", 1170 "__tsan_atomic32_fetch_add", 1171 "__tsan_atomic64_fetch_add", 1172 "__tsan_atomic8_fetch_sub", 1173 "__tsan_atomic16_fetch_sub", 1174 "__tsan_atomic32_fetch_sub", 1175 "__tsan_atomic64_fetch_sub", 1176 "__tsan_atomic8_fetch_and", 1177 "__tsan_atomic16_fetch_and", 1178 "__tsan_atomic32_fetch_and", 1179 "__tsan_atomic64_fetch_and", 1180 "__tsan_atomic8_fetch_or", 1181 "__tsan_atomic16_fetch_or", 1182 "__tsan_atomic32_fetch_or", 1183 "__tsan_atomic64_fetch_or", 1184 "__tsan_atomic8_fetch_xor", 1185 "__tsan_atomic16_fetch_xor", 1186 "__tsan_atomic32_fetch_xor", 1187 "__tsan_atomic64_fetch_xor", 1188 "__tsan_atomic8_fetch_nand", 1189 "__tsan_atomic16_fetch_nand", 1190 "__tsan_atomic32_fetch_nand", 1191 "__tsan_atomic64_fetch_nand", 1192 "__tsan_atomic8_compare_exchange_strong", 1193 "__tsan_atomic16_compare_exchange_strong", 1194 "__tsan_atomic32_compare_exchange_strong", 1195 "__tsan_atomic64_compare_exchange_strong", 1196 "__tsan_atomic8_compare_exchange_weak", 1197 "__tsan_atomic16_compare_exchange_weak", 1198 "__tsan_atomic32_compare_exchange_weak", 1199 "__tsan_atomic64_compare_exchange_weak", 1200 "__tsan_atomic8_compare_exchange_val", 1201 "__tsan_atomic16_compare_exchange_val", 1202 "__tsan_atomic32_compare_exchange_val", 1203 "__tsan_atomic64_compare_exchange_val", 1204 "__tsan_atomic_thread_fence", 1205 "__tsan_atomic_signal_fence", 1206 "__tsan_unaligned_read16", 1207 "__tsan_unaligned_write16", 1208 /* KCOV */ 1209 "write_comp_data", 1210 "check_kcov_mode", 1211 "__sanitizer_cov_trace_pc", 1212 "__sanitizer_cov_trace_const_cmp1", 1213 "__sanitizer_cov_trace_const_cmp2", 1214 "__sanitizer_cov_trace_const_cmp4", 1215 "__sanitizer_cov_trace_const_cmp8", 1216 "__sanitizer_cov_trace_cmp1", 1217 "__sanitizer_cov_trace_cmp2", 1218 "__sanitizer_cov_trace_cmp4", 1219 "__sanitizer_cov_trace_cmp8", 1220 "__sanitizer_cov_trace_switch", 1221 /* KMSAN */ 1222 "kmsan_copy_to_user", 1223 "kmsan_disable_current", 1224 "kmsan_enable_current", 1225 "kmsan_report", 1226 "kmsan_unpoison_entry_regs", 1227 "kmsan_unpoison_memory", 1228 "__msan_chain_origin", 1229 "__msan_get_context_state", 1230 "__msan_instrument_asm_store", 1231 "__msan_metadata_ptr_for_load_1", 1232 "__msan_metadata_ptr_for_load_2", 1233 "__msan_metadata_ptr_for_load_4", 1234 "__msan_metadata_ptr_for_load_8", 1235 "__msan_metadata_ptr_for_load_n", 1236 "__msan_metadata_ptr_for_store_1", 1237 "__msan_metadata_ptr_for_store_2", 1238 "__msan_metadata_ptr_for_store_4", 1239 "__msan_metadata_ptr_for_store_8", 1240 "__msan_metadata_ptr_for_store_n", 1241 "__msan_poison_alloca", 1242 "__msan_warning", 1243 /* UBSAN */ 1244 "ubsan_type_mismatch_common", 1245 "__ubsan_handle_type_mismatch", 1246 "__ubsan_handle_type_mismatch_v1", 1247 "__ubsan_handle_shift_out_of_bounds", 1248 "__ubsan_handle_load_invalid_value", 1249 /* KSTACK_ERASE */ 1250 "__sanitizer_cov_stack_depth", 1251 /* TRACE_BRANCH_PROFILING */ 1252 "ftrace_likely_update", 1253 /* STACKPROTECTOR */ 1254 "__stack_chk_fail", 1255 /* misc */ 1256 "csum_partial_copy_generic", 1257 "copy_mc_fragile", 1258 "copy_mc_fragile_handle_tail", 1259 "copy_mc_enhanced_fast_string", 1260 "rep_stos_alternative", 1261 "rep_movs_alternative", 1262 "__copy_user_nocache", 1263 NULL 1264 }; 1265 1266 static void add_uaccess_safe(struct objtool_file *file) 1267 { 1268 struct symbol *func; 1269 const char **name; 1270 1271 if (!opts.uaccess) 1272 return; 1273 1274 for (name = uaccess_safe_builtin; *name; name++) { 1275 func = find_symbol_by_name(file->elf, *name); 1276 if (!func) 1277 continue; 1278 1279 func->uaccess_safe = true; 1280 } 1281 } 1282 1283 /* 1284 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol 1285 * will be added to the .retpoline_sites section. 1286 */ 1287 __weak bool arch_is_retpoline(struct symbol *sym) 1288 { 1289 return false; 1290 } 1291 1292 /* 1293 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol 1294 * will be added to the .return_sites section. 1295 */ 1296 __weak bool arch_is_rethunk(struct symbol *sym) 1297 { 1298 return false; 1299 } 1300 1301 /* 1302 * Symbols that are embedded inside other instructions, because sometimes crazy 1303 * code exists. These are mostly ignored for validation purposes. 1304 */ 1305 __weak bool arch_is_embedded_insn(struct symbol *sym) 1306 { 1307 return false; 1308 } 1309 1310 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) 1311 { 1312 struct reloc *reloc; 1313 1314 if (insn->no_reloc) 1315 return NULL; 1316 1317 if (!file) 1318 return NULL; 1319 1320 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 1321 insn->offset, insn->len); 1322 if (!reloc) { 1323 insn->no_reloc = 1; 1324 return NULL; 1325 } 1326 1327 return reloc; 1328 } 1329 1330 static void remove_insn_ops(struct instruction *insn) 1331 { 1332 struct stack_op *op, *next; 1333 1334 for (op = insn->stack_ops; op; op = next) { 1335 next = op->next; 1336 free(op); 1337 } 1338 insn->stack_ops = NULL; 1339 } 1340 1341 static int annotate_call_site(struct objtool_file *file, 1342 struct instruction *insn, bool sibling) 1343 { 1344 struct reloc *reloc = insn_reloc(file, insn); 1345 struct symbol *sym = insn_call_dest(insn); 1346 1347 if (!sym) 1348 sym = reloc->sym; 1349 1350 if (sym->static_call_tramp) { 1351 list_add_tail(&insn->call_node, &file->static_call_list); 1352 return 0; 1353 } 1354 1355 if (sym->retpoline_thunk) { 1356 list_add_tail(&insn->call_node, &file->retpoline_call_list); 1357 return 0; 1358 } 1359 1360 /* 1361 * Many compilers cannot disable KCOV or sanitizer calls with a function 1362 * attribute so they need a little help, NOP out any such calls from 1363 * noinstr text. 1364 */ 1365 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) { 1366 if (reloc) 1367 set_reloc_type(file->elf, reloc, R_NONE); 1368 1369 if (elf_write_insn(file->elf, insn->sec, 1370 insn->offset, insn->len, 1371 sibling ? arch_ret_insn(insn->len) 1372 : arch_nop_insn(insn->len))) { 1373 return -1; 1374 } 1375 1376 insn->type = sibling ? INSN_RETURN : INSN_NOP; 1377 1378 if (sibling) { 1379 /* 1380 * We've replaced the tail-call JMP insn by two new 1381 * insn: RET; INT3, except we only have a single struct 1382 * insn here. Mark it retpoline_safe to avoid the SLS 1383 * warning, instead of adding another insn. 1384 */ 1385 insn->retpoline_safe = true; 1386 } 1387 1388 return 0; 1389 } 1390 1391 if (opts.mcount && sym->fentry) { 1392 if (sibling) 1393 WARN_INSN(insn, "tail call to __fentry__ !?!?"); 1394 if (opts.mnop) { 1395 if (reloc) 1396 set_reloc_type(file->elf, reloc, R_NONE); 1397 1398 if (elf_write_insn(file->elf, insn->sec, 1399 insn->offset, insn->len, 1400 arch_nop_insn(insn->len))) { 1401 return -1; 1402 } 1403 1404 insn->type = INSN_NOP; 1405 } 1406 1407 list_add_tail(&insn->call_node, &file->mcount_loc_list); 1408 return 0; 1409 } 1410 1411 if (insn->type == INSN_CALL && !insn->sec->init && 1412 !insn->_call_dest->embedded_insn) 1413 list_add_tail(&insn->call_node, &file->call_list); 1414 1415 if (!sibling && dead_end_function(file, sym)) 1416 insn->dead_end = true; 1417 1418 return 0; 1419 } 1420 1421 static int add_call_dest(struct objtool_file *file, struct instruction *insn, 1422 struct symbol *dest, bool sibling) 1423 { 1424 insn->_call_dest = dest; 1425 if (!dest) 1426 return 0; 1427 1428 /* 1429 * Whatever stack impact regular CALLs have, should be undone 1430 * by the RETURN of the called function. 1431 * 1432 * Annotated intra-function calls retain the stack_ops but 1433 * are converted to JUMP, see read_intra_function_calls(). 1434 */ 1435 remove_insn_ops(insn); 1436 1437 return annotate_call_site(file, insn, sibling); 1438 } 1439 1440 static int add_retpoline_call(struct objtool_file *file, struct instruction *insn) 1441 { 1442 /* 1443 * Retpoline calls/jumps are really dynamic calls/jumps in disguise, 1444 * so convert them accordingly. 1445 */ 1446 switch (insn->type) { 1447 case INSN_CALL: 1448 insn->type = INSN_CALL_DYNAMIC; 1449 break; 1450 case INSN_JUMP_UNCONDITIONAL: 1451 insn->type = INSN_JUMP_DYNAMIC; 1452 break; 1453 case INSN_JUMP_CONDITIONAL: 1454 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL; 1455 break; 1456 default: 1457 return 0; 1458 } 1459 1460 insn->retpoline_safe = true; 1461 1462 /* 1463 * Whatever stack impact regular CALLs have, should be undone 1464 * by the RETURN of the called function. 1465 * 1466 * Annotated intra-function calls retain the stack_ops but 1467 * are converted to JUMP, see read_intra_function_calls(). 1468 */ 1469 remove_insn_ops(insn); 1470 1471 return annotate_call_site(file, insn, false); 1472 } 1473 1474 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) 1475 { 1476 /* 1477 * Return thunk tail calls are really just returns in disguise, 1478 * so convert them accordingly. 1479 */ 1480 insn->type = INSN_RETURN; 1481 insn->retpoline_safe = true; 1482 1483 if (add) 1484 list_add_tail(&insn->call_node, &file->return_thunk_list); 1485 } 1486 1487 static bool is_first_func_insn(struct objtool_file *file, 1488 struct instruction *insn) 1489 { 1490 struct symbol *func = insn_func(insn); 1491 1492 if (!func) 1493 return false; 1494 1495 if (insn->offset == func->offset) 1496 return true; 1497 1498 /* Allow direct CALL/JMP past ENDBR */ 1499 if (opts.ibt) { 1500 struct instruction *prev = prev_insn_same_sym(file, insn); 1501 1502 if (prev && prev->type == INSN_ENDBR && 1503 insn->offset == func->offset + prev->len) 1504 return true; 1505 } 1506 1507 return false; 1508 } 1509 1510 /* 1511 * Find the destination instructions for all jumps. 1512 */ 1513 static int add_jump_destinations(struct objtool_file *file) 1514 { 1515 struct instruction *insn; 1516 struct reloc *reloc; 1517 1518 for_each_insn(file, insn) { 1519 struct symbol *func = insn_func(insn); 1520 struct instruction *dest_insn; 1521 struct section *dest_sec; 1522 struct symbol *dest_sym; 1523 unsigned long dest_off; 1524 1525 if (!is_static_jump(insn)) 1526 continue; 1527 1528 if (insn->jump_dest) { 1529 /* 1530 * handle_group_alt() may have previously set 1531 * 'jump_dest' for some alternatives. 1532 */ 1533 continue; 1534 } 1535 1536 reloc = insn_reloc(file, insn); 1537 if (!reloc) { 1538 dest_sec = insn->sec; 1539 dest_off = arch_jump_destination(insn); 1540 dest_sym = dest_sec->sym; 1541 } else { 1542 dest_sym = reloc->sym; 1543 if (is_undef_sym(dest_sym)) { 1544 if (dest_sym->retpoline_thunk) { 1545 if (add_retpoline_call(file, insn)) 1546 return -1; 1547 continue; 1548 } 1549 1550 if (dest_sym->return_thunk) { 1551 add_return_call(file, insn, true); 1552 continue; 1553 } 1554 1555 /* External symbol */ 1556 if (func) { 1557 /* External sibling call */ 1558 if (add_call_dest(file, insn, dest_sym, true)) 1559 return -1; 1560 continue; 1561 } 1562 1563 /* Non-func asm code jumping to external symbol */ 1564 continue; 1565 } 1566 1567 dest_sec = dest_sym->sec; 1568 dest_off = dest_sym->offset + arch_insn_adjusted_addend(insn, reloc); 1569 } 1570 1571 dest_insn = find_insn(file, dest_sec, dest_off); 1572 if (!dest_insn) { 1573 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); 1574 1575 /* 1576 * retbleed_untrain_ret() jumps to 1577 * __x86_return_thunk(), but objtool can't find 1578 * the thunk's starting RET instruction, 1579 * because the RET is also in the middle of 1580 * another instruction. Objtool only knows 1581 * about the outer instruction. 1582 */ 1583 if (sym && sym->embedded_insn) { 1584 add_return_call(file, insn, false); 1585 continue; 1586 } 1587 1588 /* 1589 * GCOV/KCOV dead code can jump to the end of 1590 * the function/section. 1591 */ 1592 if (file->ignore_unreachables && func && 1593 dest_sec == insn->sec && 1594 dest_off == func->offset + func->len) 1595 continue; 1596 1597 ERROR_INSN(insn, "can't find jump dest instruction at %s", 1598 offstr(dest_sec, dest_off)); 1599 return -1; 1600 } 1601 1602 if (!dest_sym || is_sec_sym(dest_sym)) { 1603 dest_sym = dest_insn->sym; 1604 if (!dest_sym) 1605 goto set_jump_dest; 1606 } 1607 1608 if (dest_sym->retpoline_thunk && dest_insn->offset == dest_sym->offset) { 1609 if (add_retpoline_call(file, insn)) 1610 return -1; 1611 continue; 1612 } 1613 1614 if (dest_sym->return_thunk && dest_insn->offset == dest_sym->offset) { 1615 add_return_call(file, insn, true); 1616 continue; 1617 } 1618 1619 if (!insn->sym || insn->sym->pfunc == dest_sym->pfunc) 1620 goto set_jump_dest; 1621 1622 /* 1623 * Internal cross-function jump. 1624 */ 1625 1626 if (is_first_func_insn(file, dest_insn)) { 1627 /* Internal sibling call */ 1628 if (add_call_dest(file, insn, dest_sym, true)) 1629 return -1; 1630 continue; 1631 } 1632 1633 set_jump_dest: 1634 insn->jump_dest = dest_insn; 1635 } 1636 1637 return 0; 1638 } 1639 1640 static struct symbol *find_call_destination(struct section *sec, unsigned long offset) 1641 { 1642 struct symbol *call_dest; 1643 1644 call_dest = find_func_by_offset(sec, offset); 1645 if (!call_dest) 1646 call_dest = find_symbol_by_offset(sec, offset); 1647 1648 return call_dest; 1649 } 1650 1651 /* 1652 * Find the destination instructions for all calls. 1653 */ 1654 static int add_call_destinations(struct objtool_file *file) 1655 { 1656 struct instruction *insn; 1657 unsigned long dest_off; 1658 struct symbol *dest; 1659 struct reloc *reloc; 1660 1661 for_each_insn(file, insn) { 1662 struct symbol *func = insn_func(insn); 1663 if (insn->type != INSN_CALL) 1664 continue; 1665 1666 reloc = insn_reloc(file, insn); 1667 if (!reloc) { 1668 dest_off = arch_jump_destination(insn); 1669 dest = find_call_destination(insn->sec, dest_off); 1670 1671 if (add_call_dest(file, insn, dest, false)) 1672 return -1; 1673 1674 if (func && func->ignore) 1675 continue; 1676 1677 if (!insn_call_dest(insn)) { 1678 ERROR_INSN(insn, "unannotated intra-function call"); 1679 return -1; 1680 } 1681 1682 if (func && !is_func_sym(insn_call_dest(insn))) { 1683 ERROR_INSN(insn, "unsupported call to non-function"); 1684 return -1; 1685 } 1686 1687 } else if (is_sec_sym(reloc->sym)) { 1688 dest_off = arch_insn_adjusted_addend(insn, reloc); 1689 dest = find_call_destination(reloc->sym->sec, dest_off); 1690 if (!dest) { 1691 ERROR_INSN(insn, "can't find call dest symbol at %s+0x%lx", 1692 reloc->sym->sec->name, dest_off); 1693 return -1; 1694 } 1695 1696 if (add_call_dest(file, insn, dest, false)) 1697 return -1; 1698 1699 } else if (reloc->sym->retpoline_thunk) { 1700 if (add_retpoline_call(file, insn)) 1701 return -1; 1702 1703 } else { 1704 if (add_call_dest(file, insn, reloc->sym, false)) 1705 return -1; 1706 } 1707 } 1708 1709 return 0; 1710 } 1711 1712 /* 1713 * The .alternatives section requires some extra special care over and above 1714 * other special sections because alternatives are patched in place. 1715 */ 1716 static int handle_group_alt(struct objtool_file *file, 1717 struct special_alt *special_alt, 1718 struct instruction *orig_insn, 1719 struct instruction **new_insn) 1720 { 1721 struct instruction *last_new_insn = NULL, *insn, *nop = NULL; 1722 struct alt_group *orig_alt_group, *new_alt_group; 1723 unsigned long dest_off; 1724 1725 orig_alt_group = orig_insn->alt_group; 1726 if (!orig_alt_group) { 1727 struct instruction *last_orig_insn = NULL; 1728 1729 orig_alt_group = calloc(1, sizeof(*orig_alt_group)); 1730 if (!orig_alt_group) { 1731 ERROR_GLIBC("calloc"); 1732 return -1; 1733 } 1734 orig_alt_group->cfi = calloc(special_alt->orig_len, 1735 sizeof(struct cfi_state *)); 1736 if (!orig_alt_group->cfi) { 1737 ERROR_GLIBC("calloc"); 1738 return -1; 1739 } 1740 1741 insn = orig_insn; 1742 sec_for_each_insn_from(file, insn) { 1743 if (insn->offset >= special_alt->orig_off + special_alt->orig_len) 1744 break; 1745 1746 insn->alt_group = orig_alt_group; 1747 last_orig_insn = insn; 1748 } 1749 orig_alt_group->orig_group = NULL; 1750 orig_alt_group->first_insn = orig_insn; 1751 orig_alt_group->last_insn = last_orig_insn; 1752 orig_alt_group->nop = NULL; 1753 orig_alt_group->ignore = orig_insn->ignore_alts; 1754 } else { 1755 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - 1756 orig_alt_group->first_insn->offset != special_alt->orig_len) { 1757 ERROR_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d", 1758 orig_alt_group->last_insn->offset + 1759 orig_alt_group->last_insn->len - 1760 orig_alt_group->first_insn->offset, 1761 special_alt->orig_len); 1762 return -1; 1763 } 1764 } 1765 1766 new_alt_group = calloc(1, sizeof(*new_alt_group)); 1767 if (!new_alt_group) { 1768 ERROR_GLIBC("calloc"); 1769 return -1; 1770 } 1771 1772 if (special_alt->new_len < special_alt->orig_len) { 1773 /* 1774 * Insert a fake nop at the end to make the replacement 1775 * alt_group the same size as the original. This is needed to 1776 * allow propagate_alt_cfi() to do its magic. When the last 1777 * instruction affects the stack, the instruction after it (the 1778 * nop) will propagate the new state to the shared CFI array. 1779 */ 1780 nop = calloc(1, sizeof(*nop)); 1781 if (!nop) { 1782 ERROR_GLIBC("calloc"); 1783 return -1; 1784 } 1785 memset(nop, 0, sizeof(*nop)); 1786 1787 nop->sec = special_alt->new_sec; 1788 nop->offset = special_alt->new_off + special_alt->new_len; 1789 nop->len = special_alt->orig_len - special_alt->new_len; 1790 nop->type = INSN_NOP; 1791 nop->sym = orig_insn->sym; 1792 nop->alt_group = new_alt_group; 1793 nop->fake = 1; 1794 } 1795 1796 if (!special_alt->new_len) { 1797 *new_insn = nop; 1798 goto end; 1799 } 1800 1801 insn = *new_insn; 1802 sec_for_each_insn_from(file, insn) { 1803 struct reloc *alt_reloc; 1804 1805 if (insn->offset >= special_alt->new_off + special_alt->new_len) 1806 break; 1807 1808 last_new_insn = insn; 1809 1810 insn->sym = orig_insn->sym; 1811 insn->alt_group = new_alt_group; 1812 1813 /* 1814 * Since alternative replacement code is copy/pasted by the 1815 * kernel after applying relocations, generally such code can't 1816 * have relative-address relocation references to outside the 1817 * .altinstr_replacement section, unless the arch's 1818 * alternatives code can adjust the relative offsets 1819 * accordingly. 1820 */ 1821 alt_reloc = insn_reloc(file, insn); 1822 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) && 1823 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { 1824 1825 ERROR_INSN(insn, "unsupported relocation in alternatives section"); 1826 return -1; 1827 } 1828 1829 if (!is_static_jump(insn)) 1830 continue; 1831 1832 if (!insn->immediate) 1833 continue; 1834 1835 dest_off = arch_jump_destination(insn); 1836 if (dest_off == special_alt->new_off + special_alt->new_len) { 1837 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn); 1838 if (!insn->jump_dest) { 1839 ERROR_INSN(insn, "can't find alternative jump destination"); 1840 return -1; 1841 } 1842 } 1843 } 1844 1845 if (!last_new_insn) { 1846 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1847 "can't find last new alternative instruction"); 1848 return -1; 1849 } 1850 1851 end: 1852 new_alt_group->orig_group = orig_alt_group; 1853 new_alt_group->first_insn = *new_insn; 1854 new_alt_group->last_insn = last_new_insn; 1855 new_alt_group->nop = nop; 1856 new_alt_group->ignore = (*new_insn)->ignore_alts; 1857 new_alt_group->cfi = orig_alt_group->cfi; 1858 return 0; 1859 } 1860 1861 /* 1862 * A jump table entry can either convert a nop to a jump or a jump to a nop. 1863 * If the original instruction is a jump, make the alt entry an effective nop 1864 * by just skipping the original instruction. 1865 */ 1866 static int handle_jump_alt(struct objtool_file *file, 1867 struct special_alt *special_alt, 1868 struct instruction *orig_insn, 1869 struct instruction **new_insn) 1870 { 1871 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && 1872 orig_insn->type != INSN_NOP) { 1873 1874 ERROR_INSN(orig_insn, "unsupported instruction at jump label"); 1875 return -1; 1876 } 1877 1878 if (opts.hack_jump_label && special_alt->key_addend & 2) { 1879 struct reloc *reloc = insn_reloc(file, orig_insn); 1880 1881 if (reloc) 1882 set_reloc_type(file->elf, reloc, R_NONE); 1883 1884 if (elf_write_insn(file->elf, orig_insn->sec, 1885 orig_insn->offset, orig_insn->len, 1886 arch_nop_insn(orig_insn->len))) { 1887 return -1; 1888 } 1889 1890 orig_insn->type = INSN_NOP; 1891 } 1892 1893 if (orig_insn->type == INSN_NOP) { 1894 if (orig_insn->len == 2) 1895 file->jl_nop_short++; 1896 else 1897 file->jl_nop_long++; 1898 1899 return 0; 1900 } 1901 1902 if (orig_insn->len == 2) 1903 file->jl_short++; 1904 else 1905 file->jl_long++; 1906 1907 *new_insn = next_insn_same_sec(file, orig_insn); 1908 return 0; 1909 } 1910 1911 /* 1912 * Read all the special sections which have alternate instructions which can be 1913 * patched in or redirected to at runtime. Each instruction having alternate 1914 * instruction(s) has them added to its insn->alts list, which will be 1915 * traversed in validate_branch(). 1916 */ 1917 static int add_special_section_alts(struct objtool_file *file) 1918 { 1919 struct list_head special_alts; 1920 struct instruction *orig_insn, *new_insn; 1921 struct special_alt *special_alt, *tmp; 1922 enum alternative_type alt_type; 1923 struct alternative *alt; 1924 1925 if (special_get_alts(file->elf, &special_alts)) 1926 return -1; 1927 1928 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) { 1929 1930 orig_insn = find_insn(file, special_alt->orig_sec, 1931 special_alt->orig_off); 1932 if (!orig_insn) { 1933 ERROR_FUNC(special_alt->orig_sec, special_alt->orig_off, 1934 "special: can't find orig instruction"); 1935 return -1; 1936 } 1937 1938 new_insn = NULL; 1939 if (!special_alt->group || special_alt->new_len) { 1940 new_insn = find_insn(file, special_alt->new_sec, 1941 special_alt->new_off); 1942 if (!new_insn) { 1943 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1944 "special: can't find new instruction"); 1945 return -1; 1946 } 1947 } 1948 1949 if (special_alt->group) { 1950 if (!special_alt->orig_len) { 1951 ERROR_INSN(orig_insn, "empty alternative entry"); 1952 continue; 1953 } 1954 1955 if (handle_group_alt(file, special_alt, orig_insn, &new_insn)) 1956 return -1; 1957 1958 alt_type = ALT_TYPE_INSTRUCTIONS; 1959 1960 } else if (special_alt->jump_or_nop) { 1961 if (handle_jump_alt(file, special_alt, orig_insn, &new_insn)) 1962 return -1; 1963 1964 alt_type = ALT_TYPE_JUMP_TABLE; 1965 } else { 1966 alt_type = ALT_TYPE_EX_TABLE; 1967 } 1968 1969 alt = calloc(1, sizeof(*alt)); 1970 if (!alt) { 1971 ERROR_GLIBC("calloc"); 1972 return -1; 1973 } 1974 1975 alt->insn = new_insn; 1976 alt->next = orig_insn->alts; 1977 alt->type = alt_type; 1978 orig_insn->alts = alt; 1979 1980 list_del(&special_alt->list); 1981 free(special_alt); 1982 } 1983 1984 if (opts.stats) { 1985 printf("jl\\\tNOP\tJMP\n"); 1986 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); 1987 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); 1988 } 1989 1990 return 0; 1991 } 1992 1993 __weak unsigned long arch_jump_table_sym_offset(struct reloc *reloc, struct reloc *table) 1994 { 1995 return reloc->sym->offset + reloc_addend(reloc); 1996 } 1997 1998 static int add_jump_table(struct objtool_file *file, struct instruction *insn) 1999 { 2000 unsigned long table_size = insn_jump_table_size(insn); 2001 struct symbol *pfunc = insn_func(insn)->pfunc; 2002 struct reloc *table = insn_jump_table(insn); 2003 struct instruction *dest_insn; 2004 unsigned int prev_offset = 0; 2005 struct reloc *reloc = table; 2006 struct alternative *alt; 2007 unsigned long sym_offset; 2008 2009 /* 2010 * Each @reloc is a switch table relocation which points to the target 2011 * instruction. 2012 */ 2013 for_each_reloc_from(table->sec, reloc) { 2014 2015 /* Check for the end of the table: */ 2016 if (table_size && reloc_offset(reloc) - reloc_offset(table) >= table_size) 2017 break; 2018 if (reloc != table && is_jump_table(reloc)) 2019 break; 2020 2021 /* Make sure the table entries are consecutive: */ 2022 if (prev_offset && reloc_offset(reloc) != prev_offset + arch_reloc_size(reloc)) 2023 break; 2024 2025 sym_offset = arch_jump_table_sym_offset(reloc, table); 2026 2027 /* Detect function pointers from contiguous objects: */ 2028 if (reloc->sym->sec == pfunc->sec && sym_offset == pfunc->offset) 2029 break; 2030 2031 /* 2032 * Clang sometimes leaves dangling unused jump table entries 2033 * which point to the end of the function. Ignore them. 2034 */ 2035 if (reloc->sym->sec == pfunc->sec && 2036 sym_offset == pfunc->offset + pfunc->len) 2037 goto next; 2038 2039 dest_insn = find_insn(file, reloc->sym->sec, sym_offset); 2040 if (!dest_insn) 2041 break; 2042 2043 /* Make sure the destination is in the same function: */ 2044 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc) 2045 break; 2046 2047 alt = calloc(1, sizeof(*alt)); 2048 if (!alt) { 2049 ERROR_GLIBC("calloc"); 2050 return -1; 2051 } 2052 2053 alt->insn = dest_insn; 2054 alt->next = insn->alts; 2055 insn->alts = alt; 2056 next: 2057 prev_offset = reloc_offset(reloc); 2058 } 2059 2060 if (!prev_offset) { 2061 ERROR_INSN(insn, "can't find switch jump table"); 2062 return -1; 2063 } 2064 2065 return 0; 2066 } 2067 2068 /* 2069 * find_jump_table() - Given a dynamic jump, find the switch jump table 2070 * associated with it. 2071 */ 2072 static void find_jump_table(struct objtool_file *file, struct symbol *func, 2073 struct instruction *insn) 2074 { 2075 struct reloc *table_reloc; 2076 struct instruction *dest_insn, *orig_insn = insn; 2077 unsigned long table_size; 2078 unsigned long sym_offset; 2079 2080 /* 2081 * Backward search using the @first_jump_src links, these help avoid 2082 * much of the 'in between' code. Which avoids us getting confused by 2083 * it. 2084 */ 2085 for (; 2086 insn && insn_func(insn) && insn_func(insn)->pfunc == func; 2087 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) { 2088 2089 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC) 2090 break; 2091 2092 /* allow small jumps within the range */ 2093 if (insn->type == INSN_JUMP_UNCONDITIONAL && 2094 insn->jump_dest && 2095 (insn->jump_dest->offset <= insn->offset || 2096 insn->jump_dest->offset > orig_insn->offset)) 2097 break; 2098 2099 table_reloc = arch_find_switch_table(file, insn, &table_size); 2100 if (!table_reloc) 2101 continue; 2102 2103 sym_offset = table_reloc->sym->offset + reloc_addend(table_reloc); 2104 2105 dest_insn = find_insn(file, table_reloc->sym->sec, sym_offset); 2106 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func) 2107 continue; 2108 2109 set_jump_table(table_reloc); 2110 orig_insn->_jump_table = table_reloc; 2111 orig_insn->_jump_table_size = table_size; 2112 2113 break; 2114 } 2115 } 2116 2117 /* 2118 * First pass: Mark the head of each jump table so that in the next pass, 2119 * we know when a given jump table ends and the next one starts. 2120 */ 2121 static void mark_func_jump_tables(struct objtool_file *file, 2122 struct symbol *func) 2123 { 2124 struct instruction *insn, *last = NULL; 2125 2126 func_for_each_insn(file, func, insn) { 2127 if (!last) 2128 last = insn; 2129 2130 /* 2131 * Store back-pointers for unconditional forward jumps such 2132 * that find_jump_table() can back-track using those and 2133 * avoid some potentially confusing code. 2134 */ 2135 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && 2136 insn->offset > last->offset && 2137 insn->jump_dest->offset > insn->offset && 2138 !insn->jump_dest->first_jump_src) { 2139 2140 insn->jump_dest->first_jump_src = insn; 2141 last = insn->jump_dest; 2142 } 2143 2144 if (insn->type != INSN_JUMP_DYNAMIC) 2145 continue; 2146 2147 find_jump_table(file, func, insn); 2148 } 2149 } 2150 2151 static int add_func_jump_tables(struct objtool_file *file, 2152 struct symbol *func) 2153 { 2154 struct instruction *insn; 2155 2156 func_for_each_insn(file, func, insn) { 2157 if (!insn_jump_table(insn)) 2158 continue; 2159 2160 if (add_jump_table(file, insn)) 2161 return -1; 2162 } 2163 2164 return 0; 2165 } 2166 2167 /* 2168 * For some switch statements, gcc generates a jump table in the .rodata 2169 * section which contains a list of addresses within the function to jump to. 2170 * This finds these jump tables and adds them to the insn->alts lists. 2171 */ 2172 static int add_jump_table_alts(struct objtool_file *file) 2173 { 2174 struct symbol *func; 2175 2176 if (!file->rodata) 2177 return 0; 2178 2179 for_each_sym(file->elf, func) { 2180 if (!is_func_sym(func) || func->alias != func) 2181 continue; 2182 2183 mark_func_jump_tables(file, func); 2184 if (add_func_jump_tables(file, func)) 2185 return -1; 2186 } 2187 2188 return 0; 2189 } 2190 2191 static void set_func_state(struct cfi_state *state) 2192 { 2193 state->cfa = initial_func_cfi.cfa; 2194 memcpy(&state->regs, &initial_func_cfi.regs, 2195 CFI_NUM_REGS * sizeof(struct cfi_reg)); 2196 state->stack_size = initial_func_cfi.cfa.offset; 2197 state->type = UNWIND_HINT_TYPE_CALL; 2198 } 2199 2200 static int read_unwind_hints(struct objtool_file *file) 2201 { 2202 struct cfi_state cfi = init_cfi; 2203 struct section *sec; 2204 struct unwind_hint *hint; 2205 struct instruction *insn; 2206 struct reloc *reloc; 2207 unsigned long offset; 2208 int i; 2209 2210 sec = find_section_by_name(file->elf, ".discard.unwind_hints"); 2211 if (!sec) 2212 return 0; 2213 2214 if (!sec->rsec) { 2215 ERROR("missing .rela.discard.unwind_hints section"); 2216 return -1; 2217 } 2218 2219 if (sec_size(sec) % sizeof(struct unwind_hint)) { 2220 ERROR("struct unwind_hint size mismatch"); 2221 return -1; 2222 } 2223 2224 file->hints = true; 2225 2226 for (i = 0; i < sec_size(sec) / sizeof(struct unwind_hint); i++) { 2227 hint = (struct unwind_hint *)sec->data->d_buf + i; 2228 2229 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); 2230 if (!reloc) { 2231 ERROR("can't find reloc for unwind_hints[%d]", i); 2232 return -1; 2233 } 2234 2235 offset = reloc->sym->offset + reloc_addend(reloc); 2236 2237 insn = find_insn(file, reloc->sym->sec, offset); 2238 if (!insn) { 2239 ERROR("can't find insn for unwind_hints[%d]", i); 2240 return -1; 2241 } 2242 2243 insn->hint = true; 2244 2245 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) { 2246 insn->cfi = &force_undefined_cfi; 2247 continue; 2248 } 2249 2250 if (hint->type == UNWIND_HINT_TYPE_SAVE) { 2251 insn->hint = false; 2252 insn->save = true; 2253 continue; 2254 } 2255 2256 if (hint->type == UNWIND_HINT_TYPE_RESTORE) { 2257 insn->restore = true; 2258 continue; 2259 } 2260 2261 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 2262 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 2263 2264 if (sym && is_global_sym(sym)) { 2265 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { 2266 ERROR_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR"); 2267 return -1; 2268 } 2269 } 2270 } 2271 2272 if (hint->type == UNWIND_HINT_TYPE_FUNC) { 2273 insn->cfi = &func_cfi; 2274 continue; 2275 } 2276 2277 if (insn->cfi) 2278 cfi = *(insn->cfi); 2279 2280 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { 2281 ERROR_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg); 2282 return -1; 2283 } 2284 2285 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); 2286 cfi.type = hint->type; 2287 cfi.signal = hint->signal; 2288 2289 insn->cfi = cfi_hash_find_or_add(&cfi); 2290 } 2291 2292 return 0; 2293 } 2294 2295 static int read_annotate(struct objtool_file *file, 2296 int (*func)(struct objtool_file *file, int type, struct instruction *insn)) 2297 { 2298 struct section *sec; 2299 struct instruction *insn; 2300 struct reloc *reloc; 2301 uint64_t offset; 2302 int type; 2303 2304 sec = find_section_by_name(file->elf, ".discard.annotate_insn"); 2305 if (!sec) 2306 return 0; 2307 2308 if (!sec->rsec) 2309 return 0; 2310 2311 if (sec->sh.sh_entsize != 8) { 2312 static bool warned = false; 2313 if (!warned && opts.verbose) { 2314 WARN("%s: dodgy linker, sh_entsize != 8", sec->name); 2315 warned = true; 2316 } 2317 sec->sh.sh_entsize = 8; 2318 } 2319 2320 if (sec_num_entries(sec) != sec_num_entries(sec->rsec)) { 2321 ERROR("bad .discard.annotate_insn section: missing relocs"); 2322 return -1; 2323 } 2324 2325 for_each_reloc(sec->rsec, reloc) { 2326 type = annotype(file->elf, sec, reloc); 2327 offset = reloc->sym->offset + reloc_addend(reloc); 2328 insn = find_insn(file, reloc->sym->sec, offset); 2329 2330 if (!insn) { 2331 ERROR("bad .discard.annotate_insn entry: %d of type %d", reloc_idx(reloc), type); 2332 return -1; 2333 } 2334 2335 if (func(file, type, insn)) 2336 return -1; 2337 } 2338 2339 return 0; 2340 } 2341 2342 static int __annotate_early(struct objtool_file *file, int type, struct instruction *insn) 2343 { 2344 switch (type) { 2345 2346 /* Must be before add_special_section_alts() */ 2347 case ANNOTYPE_IGNORE_ALTS: 2348 insn->ignore_alts = true; 2349 break; 2350 2351 /* 2352 * Must be before read_unwind_hints() since that needs insn->noendbr. 2353 */ 2354 case ANNOTYPE_NOENDBR: 2355 insn->noendbr = 1; 2356 break; 2357 2358 default: 2359 break; 2360 } 2361 2362 return 0; 2363 } 2364 2365 static int __annotate_ifc(struct objtool_file *file, int type, struct instruction *insn) 2366 { 2367 unsigned long dest_off; 2368 2369 if (type != ANNOTYPE_INTRA_FUNCTION_CALL) 2370 return 0; 2371 2372 if (insn->type != INSN_CALL) { 2373 ERROR_INSN(insn, "intra_function_call not a direct call"); 2374 return -1; 2375 } 2376 2377 /* 2378 * Treat intra-function CALLs as JMPs, but with a stack_op. 2379 * See add_call_destinations(), which strips stack_ops from 2380 * normal CALLs. 2381 */ 2382 insn->type = INSN_JUMP_UNCONDITIONAL; 2383 2384 dest_off = arch_jump_destination(insn); 2385 insn->jump_dest = find_insn(file, insn->sec, dest_off); 2386 if (!insn->jump_dest) { 2387 ERROR_INSN(insn, "can't find call dest at %s+0x%lx", 2388 insn->sec->name, dest_off); 2389 return -1; 2390 } 2391 2392 return 0; 2393 } 2394 2395 static int __annotate_late(struct objtool_file *file, int type, struct instruction *insn) 2396 { 2397 struct symbol *sym; 2398 2399 switch (type) { 2400 case ANNOTYPE_NOENDBR: 2401 /* early */ 2402 break; 2403 2404 case ANNOTYPE_RETPOLINE_SAFE: 2405 if (insn->type != INSN_JUMP_DYNAMIC && 2406 insn->type != INSN_CALL_DYNAMIC && 2407 insn->type != INSN_RETURN && 2408 insn->type != INSN_NOP) { 2409 ERROR_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); 2410 return -1; 2411 } 2412 2413 insn->retpoline_safe = true; 2414 break; 2415 2416 case ANNOTYPE_INSTR_BEGIN: 2417 insn->instr++; 2418 break; 2419 2420 case ANNOTYPE_INSTR_END: 2421 insn->instr--; 2422 break; 2423 2424 case ANNOTYPE_UNRET_BEGIN: 2425 insn->unret = 1; 2426 break; 2427 2428 case ANNOTYPE_IGNORE_ALTS: 2429 /* early */ 2430 break; 2431 2432 case ANNOTYPE_INTRA_FUNCTION_CALL: 2433 /* ifc */ 2434 break; 2435 2436 case ANNOTYPE_REACHABLE: 2437 insn->dead_end = false; 2438 break; 2439 2440 case ANNOTYPE_NOCFI: 2441 sym = insn->sym; 2442 if (!sym) { 2443 ERROR_INSN(insn, "dodgy NOCFI annotation"); 2444 return -1; 2445 } 2446 insn->sym->nocfi = 1; 2447 break; 2448 2449 default: 2450 ERROR_INSN(insn, "Unknown annotation type: %d", type); 2451 return -1; 2452 } 2453 2454 return 0; 2455 } 2456 2457 /* 2458 * Return true if name matches an instrumentation function, where calls to that 2459 * function from noinstr code can safely be removed, but compilers won't do so. 2460 */ 2461 static bool is_profiling_func(const char *name) 2462 { 2463 /* 2464 * Many compilers cannot disable KCOV with a function attribute. 2465 */ 2466 if (!strncmp(name, "__sanitizer_cov_", 16)) 2467 return true; 2468 2469 return false; 2470 } 2471 2472 static int classify_symbols(struct objtool_file *file) 2473 { 2474 struct symbol *func; 2475 size_t len; 2476 2477 for_each_sym(file->elf, func) { 2478 if (is_notype_sym(func) && strstarts(func->name, ".L")) 2479 func->local_label = true; 2480 2481 if (!is_global_sym(func)) 2482 continue; 2483 2484 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, 2485 strlen(STATIC_CALL_TRAMP_PREFIX_STR))) 2486 func->static_call_tramp = true; 2487 2488 if (arch_is_retpoline(func)) 2489 func->retpoline_thunk = true; 2490 2491 if (arch_is_rethunk(func)) 2492 func->return_thunk = true; 2493 2494 if (arch_is_embedded_insn(func)) 2495 func->embedded_insn = true; 2496 2497 if (arch_ftrace_match(func->name)) 2498 func->fentry = true; 2499 2500 if (is_profiling_func(func->name)) 2501 func->profiling_func = true; 2502 2503 len = strlen(func->name); 2504 if (len > sym_name_max_len) 2505 sym_name_max_len = len; 2506 } 2507 2508 return 0; 2509 } 2510 2511 static void mark_rodata(struct objtool_file *file) 2512 { 2513 struct section *sec; 2514 bool found = false; 2515 2516 /* 2517 * Search for the following rodata sections, each of which can 2518 * potentially contain jump tables: 2519 * 2520 * - .rodata: can contain GCC switch tables 2521 * - .rodata.<func>: same, if -fdata-sections is being used 2522 * - .data.rel.ro.c_jump_table: contains C annotated jump tables 2523 * 2524 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2525 */ 2526 for_each_sec(file->elf, sec) { 2527 if ((!strncmp(sec->name, ".rodata", 7) && 2528 !strstr(sec->name, ".str1.")) || 2529 !strncmp(sec->name, ".data.rel.ro", 12)) { 2530 sec->rodata = true; 2531 found = true; 2532 } 2533 } 2534 2535 file->rodata = found; 2536 } 2537 2538 static void mark_holes(struct objtool_file *file) 2539 { 2540 struct instruction *insn; 2541 bool in_hole = false; 2542 2543 if (!opts.link) 2544 return; 2545 2546 /* 2547 * Whole archive runs might encounter dead code from weak symbols. 2548 * This is where the linker will have dropped the weak symbol in 2549 * favour of a regular symbol, but leaves the code in place. 2550 */ 2551 for_each_insn(file, insn) { 2552 if (insn->sym || !find_symbol_hole_containing(insn->sec, insn->offset)) { 2553 in_hole = false; 2554 continue; 2555 } 2556 2557 /* Skip function padding and pfx code */ 2558 if (!in_hole && insn->type == INSN_NOP) 2559 continue; 2560 2561 in_hole = true; 2562 insn->hole = 1; 2563 2564 /* 2565 * If this hole jumps to a .cold function, mark it ignore. 2566 */ 2567 if (insn->jump_dest) { 2568 struct symbol *dest_func = insn_func(insn->jump_dest); 2569 2570 if (dest_func && dest_func->cold) 2571 dest_func->ignore = true; 2572 } 2573 } 2574 } 2575 2576 static bool validate_branch_enabled(void) 2577 { 2578 return opts.stackval || 2579 opts.orc || 2580 opts.uaccess || 2581 opts.checksum; 2582 } 2583 2584 static int decode_sections(struct objtool_file *file) 2585 { 2586 file->klp = is_livepatch_module(file); 2587 2588 mark_rodata(file); 2589 2590 if (init_pv_ops(file)) 2591 return -1; 2592 2593 /* 2594 * Must be before add_{jump_call}_destination. 2595 */ 2596 if (classify_symbols(file)) 2597 return -1; 2598 2599 if (decode_instructions(file)) 2600 return -1; 2601 2602 if (add_ignores(file)) 2603 return -1; 2604 2605 add_uaccess_safe(file); 2606 2607 if (read_annotate(file, __annotate_early)) 2608 return -1; 2609 2610 /* 2611 * Must be before add_jump_destinations(), which depends on 'func' 2612 * being set for alternatives, to enable proper sibling call detection. 2613 */ 2614 if (validate_branch_enabled() || opts.noinstr || opts.hack_jump_label) { 2615 if (add_special_section_alts(file)) 2616 return -1; 2617 } 2618 2619 if (add_jump_destinations(file)) 2620 return -1; 2621 2622 /* 2623 * Must be before add_call_destination(); it changes INSN_CALL to 2624 * INSN_JUMP. 2625 */ 2626 if (read_annotate(file, __annotate_ifc)) 2627 return -1; 2628 2629 if (add_call_destinations(file)) 2630 return -1; 2631 2632 if (add_jump_table_alts(file)) 2633 return -1; 2634 2635 if (read_unwind_hints(file)) 2636 return -1; 2637 2638 /* Must be after add_jump_destinations() */ 2639 mark_holes(file); 2640 2641 /* 2642 * Must be after add_call_destinations() such that it can override 2643 * dead_end_function() marks. 2644 */ 2645 if (read_annotate(file, __annotate_late)) 2646 return -1; 2647 2648 return 0; 2649 } 2650 2651 static bool is_special_call(struct instruction *insn) 2652 { 2653 if (insn->type == INSN_CALL) { 2654 struct symbol *dest = insn_call_dest(insn); 2655 2656 if (!dest) 2657 return false; 2658 2659 if (dest->fentry || dest->embedded_insn) 2660 return true; 2661 } 2662 2663 return false; 2664 } 2665 2666 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state) 2667 { 2668 struct cfi_state *cfi = &state->cfi; 2669 int i; 2670 2671 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap) 2672 return true; 2673 2674 if (cfi->cfa.offset != initial_func_cfi.cfa.offset) 2675 return true; 2676 2677 if (cfi->stack_size != initial_func_cfi.cfa.offset) 2678 return true; 2679 2680 for (i = 0; i < CFI_NUM_REGS; i++) { 2681 if (cfi->regs[i].base != initial_func_cfi.regs[i].base || 2682 cfi->regs[i].offset != initial_func_cfi.regs[i].offset) 2683 return true; 2684 } 2685 2686 return false; 2687 } 2688 2689 static bool check_reg_frame_pos(const struct cfi_reg *reg, 2690 int expected_offset) 2691 { 2692 return reg->base == CFI_CFA && 2693 reg->offset == expected_offset; 2694 } 2695 2696 static bool has_valid_stack_frame(struct insn_state *state) 2697 { 2698 struct cfi_state *cfi = &state->cfi; 2699 2700 if (cfi->cfa.base == CFI_BP && 2701 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) && 2702 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8)) 2703 return true; 2704 2705 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP) 2706 return true; 2707 2708 return false; 2709 } 2710 2711 static int update_cfi_state_regs(struct instruction *insn, 2712 struct cfi_state *cfi, 2713 struct stack_op *op) 2714 { 2715 struct cfi_reg *cfa = &cfi->cfa; 2716 2717 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT) 2718 return 0; 2719 2720 /* push */ 2721 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF) 2722 cfa->offset += 8; 2723 2724 /* pop */ 2725 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF) 2726 cfa->offset -= 8; 2727 2728 /* add immediate to sp */ 2729 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD && 2730 op->dest.reg == CFI_SP && op->src.reg == CFI_SP) 2731 cfa->offset -= op->src.offset; 2732 2733 return 0; 2734 } 2735 2736 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset) 2737 { 2738 if (arch_callee_saved_reg(reg) && 2739 cfi->regs[reg].base == CFI_UNDEFINED) { 2740 cfi->regs[reg].base = base; 2741 cfi->regs[reg].offset = offset; 2742 } 2743 } 2744 2745 static void restore_reg(struct cfi_state *cfi, unsigned char reg) 2746 { 2747 cfi->regs[reg].base = initial_func_cfi.regs[reg].base; 2748 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset; 2749 } 2750 2751 /* 2752 * A note about DRAP stack alignment: 2753 * 2754 * GCC has the concept of a DRAP register, which is used to help keep track of 2755 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP 2756 * register. The typical DRAP pattern is: 2757 * 2758 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10 2759 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp 2760 * 41 ff 72 f8 pushq -0x8(%r10) 2761 * 55 push %rbp 2762 * 48 89 e5 mov %rsp,%rbp 2763 * (more pushes) 2764 * 41 52 push %r10 2765 * ... 2766 * 41 5a pop %r10 2767 * (more pops) 2768 * 5d pop %rbp 2769 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2770 * c3 retq 2771 * 2772 * There are some variations in the epilogues, like: 2773 * 2774 * 5b pop %rbx 2775 * 41 5a pop %r10 2776 * 41 5c pop %r12 2777 * 41 5d pop %r13 2778 * 41 5e pop %r14 2779 * c9 leaveq 2780 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2781 * c3 retq 2782 * 2783 * and: 2784 * 2785 * 4c 8b 55 e8 mov -0x18(%rbp),%r10 2786 * 48 8b 5d e0 mov -0x20(%rbp),%rbx 2787 * 4c 8b 65 f0 mov -0x10(%rbp),%r12 2788 * 4c 8b 6d f8 mov -0x8(%rbp),%r13 2789 * c9 leaveq 2790 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2791 * c3 retq 2792 * 2793 * Sometimes r13 is used as the DRAP register, in which case it's saved and 2794 * restored beforehand: 2795 * 2796 * 41 55 push %r13 2797 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13 2798 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp 2799 * ... 2800 * 49 8d 65 f0 lea -0x10(%r13),%rsp 2801 * 41 5d pop %r13 2802 * c3 retq 2803 */ 2804 static int update_cfi_state(struct instruction *insn, 2805 struct instruction *next_insn, 2806 struct cfi_state *cfi, struct stack_op *op) 2807 { 2808 struct cfi_reg *cfa = &cfi->cfa; 2809 struct cfi_reg *regs = cfi->regs; 2810 2811 /* ignore UNWIND_HINT_UNDEFINED regions */ 2812 if (cfi->force_undefined) 2813 return 0; 2814 2815 /* stack operations don't make sense with an undefined CFA */ 2816 if (cfa->base == CFI_UNDEFINED) { 2817 if (insn_func(insn)) { 2818 WARN_INSN(insn, "undefined stack state"); 2819 return 1; 2820 } 2821 return 0; 2822 } 2823 2824 if (cfi->type == UNWIND_HINT_TYPE_REGS || 2825 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) 2826 return update_cfi_state_regs(insn, cfi, op); 2827 2828 switch (op->dest.type) { 2829 2830 case OP_DEST_REG: 2831 switch (op->src.type) { 2832 2833 case OP_SRC_REG: 2834 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP && 2835 cfa->base == CFI_SP && 2836 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) { 2837 2838 /* mov %rsp, %rbp */ 2839 cfa->base = op->dest.reg; 2840 cfi->bp_scratch = false; 2841 } 2842 2843 else if (op->src.reg == CFI_SP && 2844 op->dest.reg == CFI_BP && cfi->drap) { 2845 2846 /* drap: mov %rsp, %rbp */ 2847 regs[CFI_BP].base = CFI_BP; 2848 regs[CFI_BP].offset = -cfi->stack_size; 2849 cfi->bp_scratch = false; 2850 } 2851 2852 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2853 2854 /* 2855 * mov %rsp, %reg 2856 * 2857 * This is needed for the rare case where GCC 2858 * does: 2859 * 2860 * mov %rsp, %rax 2861 * ... 2862 * mov %rax, %rsp 2863 */ 2864 cfi->vals[op->dest.reg].base = CFI_CFA; 2865 cfi->vals[op->dest.reg].offset = -cfi->stack_size; 2866 } 2867 2868 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP && 2869 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) { 2870 2871 /* 2872 * mov %rbp, %rsp 2873 * 2874 * Restore the original stack pointer (Clang). 2875 */ 2876 cfi->stack_size = -cfi->regs[CFI_BP].offset; 2877 } 2878 2879 else if (op->dest.reg == cfa->base) { 2880 2881 /* mov %reg, %rsp */ 2882 if (cfa->base == CFI_SP && 2883 cfi->vals[op->src.reg].base == CFI_CFA) { 2884 2885 /* 2886 * This is needed for the rare case 2887 * where GCC does something dumb like: 2888 * 2889 * lea 0x8(%rsp), %rcx 2890 * ... 2891 * mov %rcx, %rsp 2892 */ 2893 cfa->offset = -cfi->vals[op->src.reg].offset; 2894 cfi->stack_size = cfa->offset; 2895 2896 } else if (cfa->base == CFI_SP && 2897 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2898 cfi->vals[op->src.reg].offset == cfa->offset) { 2899 2900 /* 2901 * Stack swizzle: 2902 * 2903 * 1: mov %rsp, (%[tos]) 2904 * 2: mov %[tos], %rsp 2905 * ... 2906 * 3: pop %rsp 2907 * 2908 * Where: 2909 * 2910 * 1 - places a pointer to the previous 2911 * stack at the Top-of-Stack of the 2912 * new stack. 2913 * 2914 * 2 - switches to the new stack. 2915 * 2916 * 3 - pops the Top-of-Stack to restore 2917 * the original stack. 2918 * 2919 * Note: we set base to SP_INDIRECT 2920 * here and preserve offset. Therefore 2921 * when the unwinder reaches ToS it 2922 * will dereference SP and then add the 2923 * offset to find the next frame, IOW: 2924 * (%rsp) + offset. 2925 */ 2926 cfa->base = CFI_SP_INDIRECT; 2927 2928 } else { 2929 cfa->base = CFI_UNDEFINED; 2930 cfa->offset = 0; 2931 } 2932 } 2933 2934 else if (op->dest.reg == CFI_SP && 2935 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2936 cfi->vals[op->src.reg].offset == cfa->offset) { 2937 2938 /* 2939 * The same stack swizzle case 2) as above. But 2940 * because we can't change cfa->base, case 3) 2941 * will become a regular POP. Pretend we're a 2942 * PUSH so things don't go unbalanced. 2943 */ 2944 cfi->stack_size += 8; 2945 } 2946 2947 2948 break; 2949 2950 case OP_SRC_ADD: 2951 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) { 2952 2953 /* add imm, %rsp */ 2954 cfi->stack_size -= op->src.offset; 2955 if (cfa->base == CFI_SP) 2956 cfa->offset -= op->src.offset; 2957 break; 2958 } 2959 2960 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP && 2961 insn->sym->frame_pointer) { 2962 /* addi.d fp,sp,imm on LoongArch */ 2963 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) { 2964 cfa->base = CFI_BP; 2965 cfa->offset = 0; 2966 } 2967 break; 2968 } 2969 2970 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { 2971 /* addi.d sp,fp,imm on LoongArch */ 2972 if (cfa->base == CFI_BP && cfa->offset == 0) { 2973 if (insn->sym->frame_pointer) { 2974 cfa->base = CFI_SP; 2975 cfa->offset = -op->src.offset; 2976 } 2977 } else { 2978 /* lea disp(%rbp), %rsp */ 2979 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); 2980 } 2981 break; 2982 } 2983 2984 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2985 2986 /* drap: lea disp(%rsp), %drap */ 2987 cfi->drap_reg = op->dest.reg; 2988 2989 /* 2990 * lea disp(%rsp), %reg 2991 * 2992 * This is needed for the rare case where GCC 2993 * does something dumb like: 2994 * 2995 * lea 0x8(%rsp), %rcx 2996 * ... 2997 * mov %rcx, %rsp 2998 */ 2999 cfi->vals[op->dest.reg].base = CFI_CFA; 3000 cfi->vals[op->dest.reg].offset = \ 3001 -cfi->stack_size + op->src.offset; 3002 3003 break; 3004 } 3005 3006 if (cfi->drap && op->dest.reg == CFI_SP && 3007 op->src.reg == cfi->drap_reg) { 3008 3009 /* drap: lea disp(%drap), %rsp */ 3010 cfa->base = CFI_SP; 3011 cfa->offset = cfi->stack_size = -op->src.offset; 3012 cfi->drap_reg = CFI_UNDEFINED; 3013 cfi->drap = false; 3014 break; 3015 } 3016 3017 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { 3018 WARN_INSN(insn, "unsupported stack register modification"); 3019 return -1; 3020 } 3021 3022 break; 3023 3024 case OP_SRC_AND: 3025 if (op->dest.reg != CFI_SP || 3026 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || 3027 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { 3028 WARN_INSN(insn, "unsupported stack pointer realignment"); 3029 return -1; 3030 } 3031 3032 if (cfi->drap_reg != CFI_UNDEFINED) { 3033 /* drap: and imm, %rsp */ 3034 cfa->base = cfi->drap_reg; 3035 cfa->offset = cfi->stack_size = 0; 3036 cfi->drap = true; 3037 } 3038 3039 /* 3040 * Older versions of GCC (4.8ish) realign the stack 3041 * without DRAP, with a frame pointer. 3042 */ 3043 3044 break; 3045 3046 case OP_SRC_POP: 3047 case OP_SRC_POPF: 3048 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) { 3049 3050 /* pop %rsp; # restore from a stack swizzle */ 3051 cfa->base = CFI_SP; 3052 break; 3053 } 3054 3055 if (!cfi->drap && op->dest.reg == cfa->base) { 3056 3057 /* pop %rbp */ 3058 cfa->base = CFI_SP; 3059 } 3060 3061 if (cfi->drap && cfa->base == CFI_BP_INDIRECT && 3062 op->dest.reg == cfi->drap_reg && 3063 cfi->drap_offset == -cfi->stack_size) { 3064 3065 /* drap: pop %drap */ 3066 cfa->base = cfi->drap_reg; 3067 cfa->offset = 0; 3068 cfi->drap_offset = -1; 3069 3070 } else if (cfi->stack_size == -regs[op->dest.reg].offset) { 3071 3072 /* pop %reg */ 3073 restore_reg(cfi, op->dest.reg); 3074 } 3075 3076 cfi->stack_size -= 8; 3077 if (cfa->base == CFI_SP) 3078 cfa->offset -= 8; 3079 3080 break; 3081 3082 case OP_SRC_REG_INDIRECT: 3083 if (!cfi->drap && op->dest.reg == cfa->base && 3084 op->dest.reg == CFI_BP) { 3085 3086 /* mov disp(%rsp), %rbp */ 3087 cfa->base = CFI_SP; 3088 cfa->offset = cfi->stack_size; 3089 } 3090 3091 if (cfi->drap && op->src.reg == CFI_BP && 3092 op->src.offset == cfi->drap_offset) { 3093 3094 /* drap: mov disp(%rbp), %drap */ 3095 cfa->base = cfi->drap_reg; 3096 cfa->offset = 0; 3097 cfi->drap_offset = -1; 3098 } 3099 3100 if (cfi->drap && op->src.reg == CFI_BP && 3101 op->src.offset == regs[op->dest.reg].offset) { 3102 3103 /* drap: mov disp(%rbp), %reg */ 3104 restore_reg(cfi, op->dest.reg); 3105 3106 } else if (op->src.reg == cfa->base && 3107 op->src.offset == regs[op->dest.reg].offset + cfa->offset) { 3108 3109 /* mov disp(%rbp), %reg */ 3110 /* mov disp(%rsp), %reg */ 3111 restore_reg(cfi, op->dest.reg); 3112 3113 } else if (op->src.reg == CFI_SP && 3114 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) { 3115 3116 /* mov disp(%rsp), %reg */ 3117 restore_reg(cfi, op->dest.reg); 3118 } 3119 3120 break; 3121 3122 default: 3123 WARN_INSN(insn, "unknown stack-related instruction"); 3124 return -1; 3125 } 3126 3127 break; 3128 3129 case OP_DEST_PUSH: 3130 case OP_DEST_PUSHF: 3131 cfi->stack_size += 8; 3132 if (cfa->base == CFI_SP) 3133 cfa->offset += 8; 3134 3135 if (op->src.type != OP_SRC_REG) 3136 break; 3137 3138 if (cfi->drap) { 3139 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3140 3141 /* drap: push %drap */ 3142 cfa->base = CFI_BP_INDIRECT; 3143 cfa->offset = -cfi->stack_size; 3144 3145 /* save drap so we know when to restore it */ 3146 cfi->drap_offset = -cfi->stack_size; 3147 3148 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) { 3149 3150 /* drap: push %rbp */ 3151 cfi->stack_size = 0; 3152 3153 } else { 3154 3155 /* drap: push %reg */ 3156 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); 3157 } 3158 3159 } else { 3160 3161 /* push %reg */ 3162 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size); 3163 } 3164 3165 /* detect when asm code uses rbp as a scratch register */ 3166 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP && 3167 cfa->base != CFI_BP) 3168 cfi->bp_scratch = true; 3169 break; 3170 3171 case OP_DEST_REG_INDIRECT: 3172 3173 if (cfi->drap) { 3174 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3175 3176 /* drap: mov %drap, disp(%rbp) */ 3177 cfa->base = CFI_BP_INDIRECT; 3178 cfa->offset = op->dest.offset; 3179 3180 /* save drap offset so we know when to restore it */ 3181 cfi->drap_offset = op->dest.offset; 3182 } else { 3183 3184 /* drap: mov reg, disp(%rbp) */ 3185 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); 3186 } 3187 3188 } else if (op->dest.reg == cfa->base) { 3189 3190 /* mov reg, disp(%rbp) */ 3191 /* mov reg, disp(%rsp) */ 3192 save_reg(cfi, op->src.reg, CFI_CFA, 3193 op->dest.offset - cfi->cfa.offset); 3194 3195 } else if (op->dest.reg == CFI_SP) { 3196 3197 /* mov reg, disp(%rsp) */ 3198 save_reg(cfi, op->src.reg, CFI_CFA, 3199 op->dest.offset - cfi->stack_size); 3200 3201 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) { 3202 3203 /* mov %rsp, (%reg); # setup a stack swizzle. */ 3204 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT; 3205 cfi->vals[op->dest.reg].offset = cfa->offset; 3206 } 3207 3208 break; 3209 3210 case OP_DEST_MEM: 3211 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { 3212 WARN_INSN(insn, "unknown stack-related memory operation"); 3213 return -1; 3214 } 3215 3216 /* pop mem */ 3217 cfi->stack_size -= 8; 3218 if (cfa->base == CFI_SP) 3219 cfa->offset -= 8; 3220 3221 break; 3222 3223 default: 3224 WARN_INSN(insn, "unknown stack-related instruction"); 3225 return -1; 3226 } 3227 3228 return 0; 3229 } 3230 3231 /* 3232 * The stack layouts of alternatives instructions can sometimes diverge when 3233 * they have stack modifications. That's fine as long as the potential stack 3234 * layouts don't conflict at any given potential instruction boundary. 3235 * 3236 * Flatten the CFIs of the different alternative code streams (both original 3237 * and replacement) into a single shared CFI array which can be used to detect 3238 * conflicts and nicely feed a linear array of ORC entries to the unwinder. 3239 */ 3240 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn) 3241 { 3242 struct cfi_state **alt_cfi; 3243 int group_off; 3244 3245 if (!insn->alt_group) 3246 return 0; 3247 3248 if (!insn->cfi) { 3249 WARN("CFI missing"); 3250 return -1; 3251 } 3252 3253 alt_cfi = insn->alt_group->cfi; 3254 group_off = insn->offset - insn->alt_group->first_insn->offset; 3255 3256 if (!alt_cfi[group_off]) { 3257 alt_cfi[group_off] = insn->cfi; 3258 } else { 3259 if (cficmp(alt_cfi[group_off], insn->cfi)) { 3260 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group; 3261 struct instruction *orig = orig_group->first_insn; 3262 WARN_INSN(orig, "stack layout conflict in alternatives: %s", 3263 offstr(insn->sec, insn->offset)); 3264 return -1; 3265 } 3266 } 3267 3268 return 0; 3269 } 3270 3271 static int handle_insn_ops(struct instruction *insn, 3272 struct instruction *next_insn, 3273 struct insn_state *state) 3274 { 3275 struct stack_op *op; 3276 int ret; 3277 3278 for (op = insn->stack_ops; op; op = op->next) { 3279 3280 ret = update_cfi_state(insn, next_insn, &state->cfi, op); 3281 if (ret) 3282 return ret; 3283 3284 if (!opts.uaccess || !insn->alt_group) 3285 continue; 3286 3287 if (op->dest.type == OP_DEST_PUSHF) { 3288 if (!state->uaccess_stack) { 3289 state->uaccess_stack = 1; 3290 } else if (state->uaccess_stack >> 31) { 3291 WARN_INSN(insn, "PUSHF stack exhausted"); 3292 return 1; 3293 } 3294 state->uaccess_stack <<= 1; 3295 state->uaccess_stack |= state->uaccess; 3296 } 3297 3298 if (op->src.type == OP_SRC_POPF) { 3299 if (state->uaccess_stack) { 3300 state->uaccess = state->uaccess_stack & 1; 3301 state->uaccess_stack >>= 1; 3302 if (state->uaccess_stack == 1) 3303 state->uaccess_stack = 0; 3304 } 3305 } 3306 } 3307 3308 return 0; 3309 } 3310 3311 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) 3312 { 3313 struct cfi_state *cfi1 = insn->cfi; 3314 int i; 3315 3316 if (!cfi1) { 3317 WARN("CFI missing"); 3318 return false; 3319 } 3320 3321 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { 3322 3323 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d", 3324 cfi1->cfa.base, cfi1->cfa.offset, 3325 cfi2->cfa.base, cfi2->cfa.offset); 3326 return false; 3327 3328 } 3329 3330 if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) { 3331 for (i = 0; i < CFI_NUM_REGS; i++) { 3332 3333 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], sizeof(struct cfi_reg))) 3334 continue; 3335 3336 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", 3337 i, cfi1->regs[i].base, cfi1->regs[i].offset, 3338 i, cfi2->regs[i].base, cfi2->regs[i].offset); 3339 } 3340 return false; 3341 } 3342 3343 if (cfi1->type != cfi2->type) { 3344 3345 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d", 3346 cfi1->type, cfi2->type); 3347 return false; 3348 } 3349 3350 if (cfi1->drap != cfi2->drap || 3351 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || 3352 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { 3353 3354 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", 3355 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, 3356 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); 3357 return false; 3358 } 3359 3360 return true; 3361 } 3362 3363 static inline bool func_uaccess_safe(struct symbol *func) 3364 { 3365 if (func) 3366 return func->uaccess_safe; 3367 3368 return false; 3369 } 3370 3371 static inline const char *call_dest_name(struct instruction *insn) 3372 { 3373 static char pvname[19]; 3374 struct reloc *reloc; 3375 int idx; 3376 3377 if (insn_call_dest(insn)) 3378 return insn_call_dest(insn)->name; 3379 3380 reloc = insn_reloc(NULL, insn); 3381 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) { 3382 idx = (reloc_addend(reloc) / sizeof(void *)); 3383 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx); 3384 return pvname; 3385 } 3386 3387 return "{dynamic}"; 3388 } 3389 3390 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn) 3391 { 3392 struct symbol *target; 3393 struct reloc *reloc; 3394 int idx; 3395 3396 reloc = insn_reloc(file, insn); 3397 if (!reloc || strcmp(reloc->sym->name, "pv_ops")) 3398 return false; 3399 3400 idx = arch_insn_adjusted_addend(insn, reloc) / sizeof(void *); 3401 3402 if (file->pv_ops[idx].clean) 3403 return true; 3404 3405 file->pv_ops[idx].clean = true; 3406 3407 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) { 3408 if (!target->sec->noinstr) { 3409 WARN("pv_ops[%d]: %s", idx, target->name); 3410 file->pv_ops[idx].clean = false; 3411 } 3412 } 3413 3414 return file->pv_ops[idx].clean; 3415 } 3416 3417 static inline bool noinstr_call_dest(struct objtool_file *file, 3418 struct instruction *insn, 3419 struct symbol *func) 3420 { 3421 /* 3422 * We can't deal with indirect function calls at present; 3423 * assume they're instrumented. 3424 */ 3425 if (!func) { 3426 if (file->pv_ops) 3427 return pv_call_dest(file, insn); 3428 3429 return false; 3430 } 3431 3432 /* 3433 * If the symbol is from a noinstr section; we good. 3434 */ 3435 if (func->sec->noinstr) 3436 return true; 3437 3438 /* 3439 * If the symbol is a static_call trampoline, we can't tell. 3440 */ 3441 if (func->static_call_tramp) 3442 return true; 3443 3444 /* 3445 * The __ubsan_handle_*() calls are like WARN(), they only happen when 3446 * something 'BAD' happened. At the risk of taking the machine down, 3447 * let them proceed to get the message out. 3448 */ 3449 if (!strncmp(func->name, "__ubsan_handle_", 15)) 3450 return true; 3451 3452 return false; 3453 } 3454 3455 static int validate_call(struct objtool_file *file, 3456 struct instruction *insn, 3457 struct insn_state *state) 3458 { 3459 if (state->noinstr && state->instr <= 0 && 3460 !noinstr_call_dest(file, insn, insn_call_dest(insn))) { 3461 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn)); 3462 return 1; 3463 } 3464 3465 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) { 3466 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn)); 3467 return 1; 3468 } 3469 3470 if (state->df) { 3471 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn)); 3472 return 1; 3473 } 3474 3475 return 0; 3476 } 3477 3478 static int validate_sibling_call(struct objtool_file *file, 3479 struct instruction *insn, 3480 struct insn_state *state) 3481 { 3482 if (insn_func(insn) && has_modified_stack_frame(insn, state)) { 3483 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame"); 3484 return 1; 3485 } 3486 3487 return validate_call(file, insn, state); 3488 } 3489 3490 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 3491 { 3492 if (state->noinstr && state->instr > 0) { 3493 WARN_INSN(insn, "return with instrumentation enabled"); 3494 return 1; 3495 } 3496 3497 if (state->uaccess && !func_uaccess_safe(func)) { 3498 WARN_INSN(insn, "return with UACCESS enabled"); 3499 return 1; 3500 } 3501 3502 if (!state->uaccess && func_uaccess_safe(func)) { 3503 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function"); 3504 return 1; 3505 } 3506 3507 if (state->df) { 3508 WARN_INSN(insn, "return with DF set"); 3509 return 1; 3510 } 3511 3512 if (func && has_modified_stack_frame(insn, state)) { 3513 WARN_INSN(insn, "return with modified stack frame"); 3514 return 1; 3515 } 3516 3517 if (state->cfi.bp_scratch) { 3518 WARN_INSN(insn, "BP used as a scratch register"); 3519 return 1; 3520 } 3521 3522 return 0; 3523 } 3524 3525 static struct instruction *next_insn_to_validate(struct objtool_file *file, 3526 struct instruction *insn) 3527 { 3528 struct alt_group *alt_group = insn->alt_group; 3529 3530 /* 3531 * Simulate the fact that alternatives are patched in-place. When the 3532 * end of a replacement alt_group is reached, redirect objtool flow to 3533 * the end of the original alt_group. 3534 * 3535 * insn->alts->insn -> alt_group->first_insn 3536 * ... 3537 * alt_group->last_insn 3538 * [alt_group->nop] -> next(orig_group->last_insn) 3539 */ 3540 if (alt_group) { 3541 if (alt_group->nop) { 3542 /* ->nop implies ->orig_group */ 3543 if (insn == alt_group->last_insn) 3544 return alt_group->nop; 3545 if (insn == alt_group->nop) 3546 goto next_orig; 3547 } 3548 if (insn == alt_group->last_insn && alt_group->orig_group) 3549 goto next_orig; 3550 } 3551 3552 return next_insn_same_sec(file, insn); 3553 3554 next_orig: 3555 return next_insn_same_sec(file, alt_group->orig_group->last_insn); 3556 } 3557 3558 static bool skip_alt_group(struct instruction *insn) 3559 { 3560 struct instruction *alt_insn = insn->alts ? insn->alts->insn : NULL; 3561 3562 if (!insn->alt_group) 3563 return false; 3564 3565 /* ANNOTATE_IGNORE_ALTERNATIVE */ 3566 if (insn->alt_group->ignore) { 3567 TRACE_ALT(insn, "alt group ignored"); 3568 return true; 3569 } 3570 3571 /* 3572 * For NOP patched with CLAC/STAC, only follow the latter to avoid 3573 * impossible code paths combining patched CLAC with unpatched STAC 3574 * or vice versa. 3575 * 3576 * ANNOTATE_IGNORE_ALTERNATIVE could have been used here, but Linus 3577 * requested not to do that to avoid hurting .s file readability 3578 * around CLAC/STAC alternative sites. 3579 */ 3580 3581 if (!alt_insn) 3582 return false; 3583 3584 /* Don't override ASM_{CLAC,STAC}_UNSAFE */ 3585 if (alt_insn->alt_group && alt_insn->alt_group->ignore) 3586 return false; 3587 3588 return alt_insn->type == INSN_CLAC || alt_insn->type == INSN_STAC; 3589 } 3590 3591 static int checksum_debug_init(struct objtool_file *file) 3592 { 3593 char *dup, *s; 3594 3595 if (!opts.debug_checksum) 3596 return 0; 3597 3598 dup = strdup(opts.debug_checksum); 3599 if (!dup) { 3600 ERROR_GLIBC("strdup"); 3601 return -1; 3602 } 3603 3604 s = dup; 3605 while (*s) { 3606 struct symbol *func; 3607 char *comma; 3608 3609 comma = strchr(s, ','); 3610 if (comma) 3611 *comma = '\0'; 3612 3613 func = find_symbol_by_name(file->elf, s); 3614 if (!func || !is_func_sym(func)) 3615 WARN("--debug-checksum: can't find '%s'", s); 3616 else 3617 func->debug_checksum = 1; 3618 3619 if (!comma) 3620 break; 3621 3622 s = comma + 1; 3623 } 3624 3625 free(dup); 3626 return 0; 3627 } 3628 3629 static void checksum_update_insn(struct objtool_file *file, struct symbol *func, 3630 struct instruction *insn) 3631 { 3632 struct reloc *reloc = insn_reloc(file, insn); 3633 unsigned long offset; 3634 struct symbol *sym; 3635 3636 if (insn->fake) 3637 return; 3638 3639 checksum_update(func, insn, insn->sec->data->d_buf + insn->offset, insn->len); 3640 3641 if (!reloc) { 3642 struct symbol *call_dest = insn_call_dest(insn); 3643 3644 if (call_dest) 3645 checksum_update(func, insn, call_dest->demangled_name, 3646 strlen(call_dest->demangled_name)); 3647 return; 3648 } 3649 3650 sym = reloc->sym; 3651 offset = arch_insn_adjusted_addend(insn, reloc); 3652 3653 if (is_string_sec(sym->sec)) { 3654 char *str; 3655 3656 str = sym->sec->data->d_buf + sym->offset + offset; 3657 checksum_update(func, insn, str, strlen(str)); 3658 return; 3659 } 3660 3661 if (is_sec_sym(sym)) { 3662 sym = find_symbol_containing(reloc->sym->sec, offset); 3663 if (!sym) 3664 return; 3665 3666 offset -= sym->offset; 3667 } 3668 3669 checksum_update(func, insn, sym->demangled_name, strlen(sym->demangled_name)); 3670 checksum_update(func, insn, &offset, sizeof(offset)); 3671 } 3672 3673 static int validate_branch(struct objtool_file *file, struct symbol *func, 3674 struct instruction *insn, struct insn_state state); 3675 static int do_validate_branch(struct objtool_file *file, struct symbol *func, 3676 struct instruction *insn, struct insn_state state); 3677 3678 static int validate_insn(struct objtool_file *file, struct symbol *func, 3679 struct instruction *insn, struct insn_state *statep, 3680 struct instruction *prev_insn, struct instruction *next_insn, 3681 bool *dead_end) 3682 { 3683 /* prev_state and alt_name are not used if there is no disassembly support */ 3684 struct insn_state prev_state __maybe_unused; 3685 char *alt_name __maybe_unused = NULL; 3686 struct alternative *alt; 3687 u8 visited; 3688 int ret; 3689 3690 /* 3691 * Any returns before the end of this function are effectively dead 3692 * ends, i.e. validate_branch() has reached the end of the branch. 3693 */ 3694 *dead_end = true; 3695 3696 visited = VISITED_BRANCH << statep->uaccess; 3697 if (insn->visited & VISITED_BRANCH_MASK) { 3698 if (!insn->hint && !insn_cfi_match(insn, &statep->cfi)) 3699 return 1; 3700 3701 if (insn->visited & visited) { 3702 TRACE_INSN(insn, "already visited"); 3703 return 0; 3704 } 3705 } else { 3706 nr_insns_visited++; 3707 } 3708 3709 if (statep->noinstr) 3710 statep->instr += insn->instr; 3711 3712 if (insn->hint) { 3713 if (insn->restore) { 3714 struct instruction *save_insn, *i; 3715 3716 i = insn; 3717 save_insn = NULL; 3718 3719 sym_for_each_insn_continue_reverse(file, func, i) { 3720 if (i->save) { 3721 save_insn = i; 3722 break; 3723 } 3724 } 3725 3726 if (!save_insn) { 3727 WARN_INSN(insn, "no corresponding CFI save for CFI restore"); 3728 return 1; 3729 } 3730 3731 if (!save_insn->visited) { 3732 /* 3733 * If the restore hint insn is at the 3734 * beginning of a basic block and was 3735 * branched to from elsewhere, and the 3736 * save insn hasn't been visited yet, 3737 * defer following this branch for now. 3738 * It will be seen later via the 3739 * straight-line path. 3740 */ 3741 if (!prev_insn) { 3742 TRACE_INSN(insn, "defer restore"); 3743 return 0; 3744 } 3745 3746 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo"); 3747 return 1; 3748 } 3749 3750 insn->cfi = save_insn->cfi; 3751 nr_cfi_reused++; 3752 } 3753 3754 statep->cfi = *insn->cfi; 3755 } else { 3756 /* XXX track if we actually changed statep->cfi */ 3757 3758 if (prev_insn && !cficmp(prev_insn->cfi, &statep->cfi)) { 3759 insn->cfi = prev_insn->cfi; 3760 nr_cfi_reused++; 3761 } else { 3762 insn->cfi = cfi_hash_find_or_add(&statep->cfi); 3763 } 3764 } 3765 3766 insn->visited |= visited; 3767 3768 if (propagate_alt_cfi(file, insn)) 3769 return 1; 3770 3771 if (insn->alts) { 3772 for (alt = insn->alts; alt; alt = alt->next) { 3773 TRACE_ALT_BEGIN(insn, alt, alt_name); 3774 ret = validate_branch(file, func, alt->insn, *statep); 3775 TRACE_ALT_END(insn, alt, alt_name); 3776 if (ret) { 3777 BT_INSN(insn, "(alt)"); 3778 return ret; 3779 } 3780 } 3781 TRACE_ALT_INFO_NOADDR(insn, "/ ", "DEFAULT"); 3782 } 3783 3784 if (skip_alt_group(insn)) 3785 return 0; 3786 3787 prev_state = *statep; 3788 ret = handle_insn_ops(insn, next_insn, statep); 3789 TRACE_INSN_STATE(insn, &prev_state, statep); 3790 3791 if (ret) 3792 return 1; 3793 3794 switch (insn->type) { 3795 3796 case INSN_RETURN: 3797 TRACE_INSN(insn, "return"); 3798 return validate_return(func, insn, statep); 3799 3800 case INSN_CALL: 3801 case INSN_CALL_DYNAMIC: 3802 if (insn->type == INSN_CALL) 3803 TRACE_INSN(insn, "call"); 3804 else 3805 TRACE_INSN(insn, "indirect call"); 3806 3807 ret = validate_call(file, insn, statep); 3808 if (ret) 3809 return ret; 3810 3811 if (opts.stackval && func && !is_special_call(insn) && 3812 !has_valid_stack_frame(statep)) { 3813 WARN_INSN(insn, "call without frame pointer save/setup"); 3814 return 1; 3815 } 3816 3817 break; 3818 3819 case INSN_JUMP_CONDITIONAL: 3820 case INSN_JUMP_UNCONDITIONAL: 3821 if (is_sibling_call(insn)) { 3822 TRACE_INSN(insn, "sibling call"); 3823 ret = validate_sibling_call(file, insn, statep); 3824 if (ret) 3825 return ret; 3826 3827 } else if (insn->jump_dest) { 3828 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3829 TRACE_INSN(insn, "unconditional jump"); 3830 else 3831 TRACE_INSN(insn, "jump taken"); 3832 3833 ret = validate_branch(file, func, insn->jump_dest, *statep); 3834 if (ret) { 3835 BT_INSN(insn, "(branch)"); 3836 return ret; 3837 } 3838 } 3839 3840 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3841 return 0; 3842 3843 TRACE_INSN(insn, "jump not taken"); 3844 break; 3845 3846 case INSN_JUMP_DYNAMIC: 3847 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3848 TRACE_INSN(insn, "indirect jump"); 3849 if (is_sibling_call(insn)) { 3850 ret = validate_sibling_call(file, insn, statep); 3851 if (ret) 3852 return ret; 3853 } 3854 3855 if (insn->type == INSN_JUMP_DYNAMIC) 3856 return 0; 3857 3858 break; 3859 3860 case INSN_SYSCALL: 3861 TRACE_INSN(insn, "syscall"); 3862 if (func && (!next_insn || !next_insn->hint)) { 3863 WARN_INSN(insn, "unsupported instruction in callable function"); 3864 return 1; 3865 } 3866 3867 break; 3868 3869 case INSN_SYSRET: 3870 TRACE_INSN(insn, "sysret"); 3871 if (func && (!next_insn || !next_insn->hint)) { 3872 WARN_INSN(insn, "unsupported instruction in callable function"); 3873 return 1; 3874 } 3875 3876 return 0; 3877 3878 case INSN_STAC: 3879 TRACE_INSN(insn, "stac"); 3880 if (!opts.uaccess) 3881 break; 3882 3883 if (statep->uaccess) { 3884 WARN_INSN(insn, "recursive UACCESS enable"); 3885 return 1; 3886 } 3887 3888 statep->uaccess = true; 3889 break; 3890 3891 case INSN_CLAC: 3892 TRACE_INSN(insn, "clac"); 3893 if (!opts.uaccess) 3894 break; 3895 3896 if (!statep->uaccess && func) { 3897 WARN_INSN(insn, "redundant UACCESS disable"); 3898 return 1; 3899 } 3900 3901 if (func_uaccess_safe(func) && !statep->uaccess_stack) { 3902 WARN_INSN(insn, "UACCESS-safe disables UACCESS"); 3903 return 1; 3904 } 3905 3906 statep->uaccess = false; 3907 break; 3908 3909 case INSN_STD: 3910 TRACE_INSN(insn, "std"); 3911 if (statep->df) { 3912 WARN_INSN(insn, "recursive STD"); 3913 return 1; 3914 } 3915 3916 statep->df = true; 3917 break; 3918 3919 case INSN_CLD: 3920 TRACE_INSN(insn, "cld"); 3921 if (!statep->df && func) { 3922 WARN_INSN(insn, "redundant CLD"); 3923 return 1; 3924 } 3925 3926 statep->df = false; 3927 break; 3928 3929 default: 3930 break; 3931 } 3932 3933 if (insn->dead_end) 3934 TRACE_INSN(insn, "dead end"); 3935 3936 *dead_end = insn->dead_end; 3937 return 0; 3938 } 3939 3940 /* 3941 * Follow the branch starting at the given instruction, and recursively follow 3942 * any other branches (jumps). Meanwhile, track the frame pointer state at 3943 * each instruction and validate all the rules described in 3944 * tools/objtool/Documentation/objtool.txt. 3945 */ 3946 static int do_validate_branch(struct objtool_file *file, struct symbol *func, 3947 struct instruction *insn, struct insn_state state) 3948 { 3949 struct instruction *next_insn, *prev_insn = NULL; 3950 bool dead_end; 3951 int ret; 3952 3953 if (func && func->ignore) 3954 return 0; 3955 3956 do { 3957 insn->trace = 0; 3958 next_insn = next_insn_to_validate(file, insn); 3959 3960 if (opts.checksum && func && insn->sec) 3961 checksum_update_insn(file, func, insn); 3962 3963 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { 3964 /* Ignore KCFI type preambles, which always fall through */ 3965 if (is_prefix_func(func)) 3966 return 0; 3967 3968 if (file->ignore_unreachables) 3969 return 0; 3970 3971 WARN("%s() falls through to next function %s()", 3972 func->name, insn_func(insn)->name); 3973 func->warned = 1; 3974 3975 return 1; 3976 } 3977 3978 ret = validate_insn(file, func, insn, &state, prev_insn, next_insn, 3979 &dead_end); 3980 3981 if (!insn->trace) { 3982 if (ret) 3983 TRACE_INSN(insn, "warning (%d)", ret); 3984 else 3985 TRACE_INSN(insn, NULL); 3986 } 3987 3988 if (!dead_end && !next_insn) { 3989 if (state.cfi.cfa.base == CFI_UNDEFINED) 3990 return 0; 3991 if (file->ignore_unreachables) 3992 return 0; 3993 3994 WARN("%s%sunexpected end of section %s", 3995 func ? func->name : "", func ? "(): " : "", 3996 insn->sec->name); 3997 return 1; 3998 } 3999 4000 prev_insn = insn; 4001 insn = next_insn; 4002 4003 } while (!dead_end); 4004 4005 return ret; 4006 } 4007 4008 static int validate_branch(struct objtool_file *file, struct symbol *func, 4009 struct instruction *insn, struct insn_state state) 4010 { 4011 int ret; 4012 4013 trace_depth_inc(); 4014 ret = do_validate_branch(file, func, insn, state); 4015 trace_depth_dec(); 4016 4017 return ret; 4018 } 4019 4020 static int validate_unwind_hint(struct objtool_file *file, 4021 struct instruction *insn, 4022 struct insn_state *state) 4023 { 4024 if (insn->hint && !insn->visited) { 4025 struct symbol *func = insn_func(insn); 4026 int ret; 4027 4028 if (opts.checksum) 4029 checksum_init(func); 4030 4031 ret = validate_branch(file, func, insn, *state); 4032 if (ret) 4033 BT_INSN(insn, "<=== (hint)"); 4034 return ret; 4035 } 4036 4037 return 0; 4038 } 4039 4040 static int validate_unwind_hints(struct objtool_file *file, struct section *sec) 4041 { 4042 struct instruction *insn; 4043 struct insn_state state; 4044 int warnings = 0; 4045 4046 if (!file->hints) 4047 return 0; 4048 4049 init_insn_state(file, &state, sec); 4050 4051 if (sec) { 4052 sec_for_each_insn(file, sec, insn) 4053 warnings += validate_unwind_hint(file, insn, &state); 4054 } else { 4055 for_each_insn(file, insn) 4056 warnings += validate_unwind_hint(file, insn, &state); 4057 } 4058 4059 return warnings; 4060 } 4061 4062 /* 4063 * Validate rethunk entry constraint: must untrain RET before the first RET. 4064 * 4065 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes 4066 * before an actual RET instruction. 4067 */ 4068 static int validate_unret(struct objtool_file *file, struct instruction *insn) 4069 { 4070 struct instruction *next, *dest; 4071 int ret; 4072 4073 for (;;) { 4074 next = next_insn_to_validate(file, insn); 4075 4076 if (insn->visited & VISITED_UNRET) 4077 return 0; 4078 4079 insn->visited |= VISITED_UNRET; 4080 4081 if (insn->alts) { 4082 struct alternative *alt; 4083 for (alt = insn->alts; alt; alt = alt->next) { 4084 ret = validate_unret(file, alt->insn); 4085 if (ret) { 4086 BT_INSN(insn, "(alt)"); 4087 return ret; 4088 } 4089 } 4090 } 4091 4092 switch (insn->type) { 4093 4094 case INSN_CALL_DYNAMIC: 4095 case INSN_JUMP_DYNAMIC: 4096 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4097 WARN_INSN(insn, "early indirect call"); 4098 return 1; 4099 4100 case INSN_JUMP_UNCONDITIONAL: 4101 case INSN_JUMP_CONDITIONAL: 4102 if (!is_sibling_call(insn)) { 4103 if (!insn->jump_dest) { 4104 WARN_INSN(insn, "unresolved jump target after linking?!?"); 4105 return 1; 4106 } 4107 ret = validate_unret(file, insn->jump_dest); 4108 if (ret) { 4109 BT_INSN(insn, "(branch%s)", 4110 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : ""); 4111 return ret; 4112 } 4113 4114 if (insn->type == INSN_JUMP_UNCONDITIONAL) 4115 return 0; 4116 4117 break; 4118 } 4119 4120 /* fallthrough */ 4121 case INSN_CALL: 4122 dest = find_insn(file, insn_call_dest(insn)->sec, 4123 insn_call_dest(insn)->offset); 4124 if (!dest) { 4125 WARN("Unresolved function after linking!?: %s", 4126 insn_call_dest(insn)->name); 4127 return 1; 4128 } 4129 4130 ret = validate_unret(file, dest); 4131 if (ret) { 4132 BT_INSN(insn, "(call)"); 4133 return ret; 4134 } 4135 /* 4136 * If a call returns without error, it must have seen UNTRAIN_RET. 4137 * Therefore any non-error return is a success. 4138 */ 4139 return 0; 4140 4141 case INSN_RETURN: 4142 WARN_INSN(insn, "RET before UNTRAIN"); 4143 return 1; 4144 4145 case INSN_SYSCALL: 4146 break; 4147 4148 case INSN_SYSRET: 4149 return 0; 4150 4151 case INSN_NOP: 4152 if (insn->retpoline_safe) 4153 return 0; 4154 break; 4155 4156 default: 4157 break; 4158 } 4159 4160 if (insn->dead_end) 4161 return 0; 4162 4163 if (!next) { 4164 WARN_INSN(insn, "teh end!"); 4165 return 1; 4166 } 4167 insn = next; 4168 } 4169 4170 return 0; 4171 } 4172 4173 /* 4174 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter 4175 * VALIDATE_UNRET_END before RET. 4176 */ 4177 static int validate_unrets(struct objtool_file *file) 4178 { 4179 struct instruction *insn; 4180 int warnings = 0; 4181 4182 for_each_insn(file, insn) { 4183 if (!insn->unret) 4184 continue; 4185 4186 warnings += validate_unret(file, insn); 4187 } 4188 4189 return warnings; 4190 } 4191 4192 static int validate_retpoline(struct objtool_file *file) 4193 { 4194 struct instruction *insn; 4195 int warnings = 0; 4196 4197 for_each_insn(file, insn) { 4198 if (insn->type != INSN_JUMP_DYNAMIC && 4199 insn->type != INSN_CALL_DYNAMIC && 4200 insn->type != INSN_RETURN) 4201 continue; 4202 4203 if (insn->retpoline_safe) 4204 continue; 4205 4206 if (insn->sec->init) 4207 continue; 4208 4209 if (insn->type == INSN_RETURN) { 4210 if (opts.rethunk) { 4211 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build"); 4212 warnings++; 4213 } 4214 continue; 4215 } 4216 4217 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build", 4218 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); 4219 warnings++; 4220 } 4221 4222 if (!opts.cfi) 4223 return warnings; 4224 4225 /* 4226 * kCFI call sites look like: 4227 * 4228 * movl $(-0x12345678), %r10d 4229 * addl -4(%r11), %r10d 4230 * jz 1f 4231 * ud2 4232 * 1: cs call __x86_indirect_thunk_r11 4233 * 4234 * Verify all indirect calls are kCFI adorned by checking for the 4235 * UD2. Notably, doing __nocfi calls to regular (cfi) functions is 4236 * broken. 4237 */ 4238 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 4239 struct symbol *sym = insn->sym; 4240 4241 if (sym && (sym->type == STT_NOTYPE || 4242 sym->type == STT_FUNC) && !sym->nocfi) { 4243 struct instruction *prev = 4244 prev_insn_same_sym(file, insn); 4245 4246 if (!prev || prev->type != INSN_BUG) { 4247 WARN_INSN(insn, "no-cfi indirect call!"); 4248 warnings++; 4249 } 4250 } 4251 } 4252 4253 return warnings; 4254 } 4255 4256 static bool is_kasan_insn(struct instruction *insn) 4257 { 4258 return (insn->type == INSN_CALL && 4259 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return")); 4260 } 4261 4262 static bool is_ubsan_insn(struct instruction *insn) 4263 { 4264 return (insn->type == INSN_CALL && 4265 !strcmp(insn_call_dest(insn)->name, 4266 "__ubsan_handle_builtin_unreachable")); 4267 } 4268 4269 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) 4270 { 4271 struct symbol *func = insn_func(insn); 4272 struct instruction *prev_insn; 4273 int i; 4274 4275 if (insn->type == INSN_NOP || insn->type == INSN_TRAP || 4276 insn->hole || (func && func->ignore)) 4277 return true; 4278 4279 /* 4280 * Ignore alternative replacement instructions. This can happen 4281 * when a whitelisted function uses one of the ALTERNATIVE macros. 4282 */ 4283 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 4284 !strcmp(insn->sec->name, ".altinstr_aux")) 4285 return true; 4286 4287 if (!func) 4288 return false; 4289 4290 if (func->static_call_tramp) 4291 return true; 4292 4293 /* 4294 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees 4295 * __builtin_unreachable(). The BUG() macro has an unreachable() after 4296 * the UD2, which causes GCC's undefined trap logic to emit another UD2 4297 * (or occasionally a JMP to UD2). 4298 * 4299 * It may also insert a UD2 after calling a __noreturn function. 4300 */ 4301 prev_insn = prev_insn_same_sec(file, insn); 4302 if (prev_insn && prev_insn->dead_end && 4303 (insn->type == INSN_BUG || 4304 (insn->type == INSN_JUMP_UNCONDITIONAL && 4305 insn->jump_dest && insn->jump_dest->type == INSN_BUG))) 4306 return true; 4307 4308 /* 4309 * Check if this (or a subsequent) instruction is related to 4310 * CONFIG_UBSAN or CONFIG_KASAN. 4311 * 4312 * End the search at 5 instructions to avoid going into the weeds. 4313 */ 4314 for (i = 0; i < 5; i++) { 4315 4316 if (is_kasan_insn(insn) || is_ubsan_insn(insn)) 4317 return true; 4318 4319 if (insn->type == INSN_JUMP_UNCONDITIONAL) { 4320 if (insn->jump_dest && 4321 insn_func(insn->jump_dest) == func) { 4322 insn = insn->jump_dest; 4323 continue; 4324 } 4325 4326 break; 4327 } 4328 4329 if (insn->offset + insn->len >= func->offset + func->len) 4330 break; 4331 4332 insn = next_insn_same_sec(file, insn); 4333 } 4334 4335 return false; 4336 } 4337 4338 /* 4339 * For FineIBT or kCFI, a certain number of bytes preceding the function may be 4340 * NOPs. Those NOPs may be rewritten at runtime and executed, so give them a 4341 * proper function name: __pfx_<func>. 4342 * 4343 * The NOPs may not exist for the following cases: 4344 * 4345 * - compiler cloned functions (*.cold, *.part0, etc) 4346 * - asm functions created with inline asm or without SYM_FUNC_START() 4347 * 4348 * Also, the function may already have a prefix from a previous objtool run 4349 * (livepatch extracted functions, or manually running objtool multiple times). 4350 * 4351 * So return 0 if the NOPs are missing or the function already has a prefix 4352 * symbol. 4353 */ 4354 static int create_prefix_symbol(struct objtool_file *file, struct symbol *func) 4355 { 4356 struct instruction *insn, *prev; 4357 char name[SYM_NAME_LEN]; 4358 struct cfi_state *cfi; 4359 4360 if (!is_func_sym(func) || is_prefix_func(func) || 4361 func->cold || func->static_call_tramp) 4362 return 0; 4363 4364 if ((strlen(func->name) + sizeof("__pfx_") > SYM_NAME_LEN)) { 4365 WARN("%s: symbol name too long, can't create __pfx_ symbol", 4366 func->name); 4367 return 0; 4368 } 4369 4370 if (snprintf_check(name, SYM_NAME_LEN, "__pfx_%s", func->name)) 4371 return -1; 4372 4373 if (file->klp) { 4374 struct symbol *pfx; 4375 4376 pfx = find_symbol_by_offset(func->sec, func->offset - opts.prefix); 4377 if (pfx && is_prefix_func(pfx) && !strcmp(pfx->name, name)) 4378 return 0; 4379 } 4380 4381 insn = find_insn(file, func->sec, func->offset); 4382 if (!insn) { 4383 WARN("%s: can't find starting instruction", func->name); 4384 return -1; 4385 } 4386 4387 for (prev = prev_insn_same_sec(file, insn); 4388 prev; 4389 prev = prev_insn_same_sec(file, prev)) { 4390 u64 offset; 4391 4392 if (prev->type != INSN_NOP) 4393 return 0; 4394 4395 offset = func->offset - prev->offset; 4396 4397 if (offset > opts.prefix) 4398 return 0; 4399 4400 if (offset < opts.prefix) 4401 continue; 4402 4403 if (!elf_create_symbol(file->elf, name, func->sec, 4404 GELF_ST_BIND(func->sym.st_info), 4405 GELF_ST_TYPE(func->sym.st_info), 4406 prev->offset, opts.prefix)) 4407 return -1; 4408 4409 break; 4410 } 4411 4412 if (!prev) 4413 return 0; 4414 4415 if (!insn->cfi) { 4416 /* 4417 * This can happen if stack validation isn't enabled or the 4418 * function is annotated with STACK_FRAME_NON_STANDARD. 4419 */ 4420 return 0; 4421 } 4422 4423 /* Propagate insn->cfi to the prefix code */ 4424 cfi = cfi_hash_find_or_add(insn->cfi); 4425 for (; prev != insn; prev = next_insn_same_sec(file, prev)) 4426 prev->cfi = cfi; 4427 4428 return 0; 4429 } 4430 4431 static int create_prefix_symbols(struct objtool_file *file) 4432 { 4433 struct section *sec; 4434 struct symbol *func; 4435 4436 for_each_sec(file->elf, sec) { 4437 if (!is_text_sec(sec)) 4438 continue; 4439 4440 sec_for_each_sym(sec, func) { 4441 if (create_prefix_symbol(file, func)) 4442 return -1; 4443 } 4444 } 4445 4446 return 0; 4447 } 4448 4449 static int validate_symbol(struct objtool_file *file, struct section *sec, 4450 struct symbol *sym, struct insn_state *state) 4451 { 4452 struct instruction *insn; 4453 struct symbol *func; 4454 int ret; 4455 4456 if (!sym->len) { 4457 WARN("%s() is missing an ELF size annotation", sym->name); 4458 return 1; 4459 } 4460 4461 if (sym->pfunc != sym || sym->alias != sym) 4462 return 0; 4463 4464 insn = find_insn(file, sec, sym->offset); 4465 if (!insn || insn->visited) 4466 return 0; 4467 4468 if (opts.uaccess) 4469 state->uaccess = sym->uaccess_safe; 4470 4471 func = insn_func(insn); 4472 4473 if (opts.checksum) 4474 checksum_init(func); 4475 4476 if (opts.trace && !fnmatch(opts.trace, sym->name, 0)) { 4477 trace_enable(); 4478 TRACE("%s: validation begin\n", sym->name); 4479 } 4480 4481 ret = validate_branch(file, func, insn, *state); 4482 if (ret) 4483 BT_INSN(insn, "<=== (sym)"); 4484 4485 TRACE("%s: validation %s\n\n", sym->name, ret ? "failed" : "end"); 4486 trace_disable(); 4487 4488 if (opts.checksum) 4489 checksum_finish(func); 4490 4491 return ret; 4492 } 4493 4494 static int validate_section(struct objtool_file *file, struct section *sec) 4495 { 4496 struct insn_state state; 4497 struct symbol *func; 4498 int warnings = 0; 4499 4500 sec_for_each_sym(sec, func) { 4501 if (!is_func_sym(func)) 4502 continue; 4503 4504 init_insn_state(file, &state, sec); 4505 set_func_state(&state.cfi); 4506 4507 warnings += validate_symbol(file, sec, func, &state); 4508 } 4509 4510 return warnings; 4511 } 4512 4513 static int validate_noinstr_sections(struct objtool_file *file) 4514 { 4515 struct section *sec; 4516 int warnings = 0; 4517 4518 sec = find_section_by_name(file->elf, ".noinstr.text"); 4519 if (sec) { 4520 warnings += validate_section(file, sec); 4521 warnings += validate_unwind_hints(file, sec); 4522 } 4523 4524 sec = find_section_by_name(file->elf, ".entry.text"); 4525 if (sec) { 4526 warnings += validate_section(file, sec); 4527 warnings += validate_unwind_hints(file, sec); 4528 } 4529 4530 sec = find_section_by_name(file->elf, ".cpuidle.text"); 4531 if (sec) { 4532 warnings += validate_section(file, sec); 4533 warnings += validate_unwind_hints(file, sec); 4534 } 4535 4536 return warnings; 4537 } 4538 4539 static int validate_functions(struct objtool_file *file) 4540 { 4541 struct section *sec; 4542 int warnings = 0; 4543 4544 for_each_sec(file->elf, sec) { 4545 if (!is_text_sec(sec)) 4546 continue; 4547 4548 warnings += validate_section(file, sec); 4549 } 4550 4551 return warnings; 4552 } 4553 4554 static void mark_endbr_used(struct instruction *insn) 4555 { 4556 if (!list_empty(&insn->call_node)) 4557 list_del_init(&insn->call_node); 4558 } 4559 4560 static bool noendbr_range(struct objtool_file *file, struct instruction *insn) 4561 { 4562 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1); 4563 struct instruction *first; 4564 4565 if (!sym) 4566 return false; 4567 4568 first = find_insn(file, sym->sec, sym->offset); 4569 if (!first) 4570 return false; 4571 4572 if (first->type != INSN_ENDBR && !first->noendbr) 4573 return false; 4574 4575 return insn->offset == sym->offset + sym->len; 4576 } 4577 4578 static int __validate_ibt_insn(struct objtool_file *file, struct instruction *insn, 4579 struct instruction *dest) 4580 { 4581 if (dest->type == INSN_ENDBR) { 4582 mark_endbr_used(dest); 4583 return 0; 4584 } 4585 4586 if (insn_func(dest) && insn_func(insn) && 4587 insn_func(dest)->pfunc == insn_func(insn)->pfunc) { 4588 /* 4589 * Anything from->to self is either _THIS_IP_ or 4590 * IRET-to-self. 4591 * 4592 * There is no sane way to annotate _THIS_IP_ since the 4593 * compiler treats the relocation as a constant and is 4594 * happy to fold in offsets, skewing any annotation we 4595 * do, leading to vast amounts of false-positives. 4596 * 4597 * There's also compiler generated _THIS_IP_ through 4598 * KCOV and such which we have no hope of annotating. 4599 * 4600 * As such, blanket accept self-references without 4601 * issue. 4602 */ 4603 return 0; 4604 } 4605 4606 /* 4607 * Accept anything ANNOTATE_NOENDBR. 4608 */ 4609 if (dest->noendbr) 4610 return 0; 4611 4612 /* 4613 * Accept if this is the instruction after a symbol 4614 * that is (no)endbr -- typical code-range usage. 4615 */ 4616 if (noendbr_range(file, dest)) 4617 return 0; 4618 4619 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4620 return 1; 4621 } 4622 4623 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn) 4624 { 4625 struct instruction *dest; 4626 struct reloc *reloc; 4627 unsigned long off; 4628 int warnings = 0; 4629 4630 /* 4631 * Looking for function pointer load relocations. Ignore 4632 * direct/indirect branches: 4633 */ 4634 switch (insn->type) { 4635 4636 case INSN_CALL: 4637 case INSN_CALL_DYNAMIC: 4638 case INSN_JUMP_CONDITIONAL: 4639 case INSN_JUMP_UNCONDITIONAL: 4640 case INSN_JUMP_DYNAMIC: 4641 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4642 case INSN_RETURN: 4643 case INSN_NOP: 4644 return 0; 4645 4646 case INSN_LEA_RIP: 4647 if (!insn_reloc(file, insn)) { 4648 /* local function pointer reference without reloc */ 4649 4650 off = arch_jump_destination(insn); 4651 4652 dest = find_insn(file, insn->sec, off); 4653 if (!dest) { 4654 WARN_INSN(insn, "corrupt function pointer reference"); 4655 return 1; 4656 } 4657 4658 return __validate_ibt_insn(file, insn, dest); 4659 } 4660 break; 4661 4662 default: 4663 break; 4664 } 4665 4666 for (reloc = insn_reloc(file, insn); 4667 reloc; 4668 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 4669 reloc_offset(reloc) + 1, 4670 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) { 4671 4672 off = reloc->sym->offset + arch_insn_adjusted_addend(insn, reloc); 4673 4674 dest = find_insn(file, reloc->sym->sec, off); 4675 if (!dest) 4676 continue; 4677 4678 warnings += __validate_ibt_insn(file, insn, dest); 4679 } 4680 4681 return warnings; 4682 } 4683 4684 static int validate_ibt_data_reloc(struct objtool_file *file, 4685 struct reloc *reloc) 4686 { 4687 struct instruction *dest; 4688 4689 dest = find_insn(file, reloc->sym->sec, 4690 reloc->sym->offset + reloc_addend(reloc)); 4691 if (!dest) 4692 return 0; 4693 4694 if (dest->type == INSN_ENDBR) { 4695 mark_endbr_used(dest); 4696 return 0; 4697 } 4698 4699 if (dest->noendbr) 4700 return 0; 4701 4702 WARN_FUNC(reloc->sec->base, reloc_offset(reloc), 4703 "data relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4704 4705 return 1; 4706 } 4707 4708 /* 4709 * Validate IBT rules and remove used ENDBR instructions from the seal list. 4710 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with 4711 * NOPs) later, in create_ibt_endbr_seal_sections(). 4712 */ 4713 static int validate_ibt(struct objtool_file *file) 4714 { 4715 struct section *sec; 4716 struct reloc *reloc; 4717 struct instruction *insn; 4718 int warnings = 0; 4719 4720 for_each_insn(file, insn) 4721 warnings += validate_ibt_insn(file, insn); 4722 4723 for_each_sec(file->elf, sec) { 4724 4725 /* Already done by validate_ibt_insn() */ 4726 if (is_text_sec(sec)) 4727 continue; 4728 4729 if (!sec->rsec) 4730 continue; 4731 4732 /* 4733 * These sections can reference text addresses, but not with 4734 * the intent to indirect branch to them. 4735 */ 4736 if ((!strncmp(sec->name, ".discard", 8) && 4737 strcmp(sec->name, ".discard.ibt_endbr_noseal")) || 4738 !strncmp(sec->name, ".debug", 6) || 4739 !strcmp(sec->name, ".altinstructions") || 4740 !strcmp(sec->name, ".ibt_endbr_seal") || 4741 !strcmp(sec->name, ".kcfi_traps") || 4742 !strcmp(sec->name, ".orc_unwind_ip") || 4743 !strcmp(sec->name, ".retpoline_sites") || 4744 !strcmp(sec->name, ".smp_locks") || 4745 !strcmp(sec->name, ".static_call_sites") || 4746 !strcmp(sec->name, "_error_injection_whitelist") || 4747 !strcmp(sec->name, "_kprobe_blacklist") || 4748 !strcmp(sec->name, "__bug_table") || 4749 !strcmp(sec->name, "__ex_table") || 4750 !strcmp(sec->name, "__jump_table") || 4751 !strcmp(sec->name, "__klp_funcs") || 4752 !strcmp(sec->name, "__mcount_loc") || 4753 !strcmp(sec->name, ".llvm.call-graph-profile") || 4754 !strcmp(sec->name, ".llvm_bb_addr_map") || 4755 !strcmp(sec->name, "__tracepoints") || 4756 !strcmp(sec->name, "__patchable_function_entries")) 4757 continue; 4758 4759 for_each_reloc(sec->rsec, reloc) 4760 warnings += validate_ibt_data_reloc(file, reloc); 4761 } 4762 4763 return warnings; 4764 } 4765 4766 static int validate_sls(struct objtool_file *file) 4767 { 4768 struct instruction *insn, *next_insn; 4769 int warnings = 0; 4770 4771 for_each_insn(file, insn) { 4772 next_insn = next_insn_same_sec(file, insn); 4773 4774 if (insn->retpoline_safe) 4775 continue; 4776 4777 switch (insn->type) { 4778 case INSN_RETURN: 4779 if (!next_insn || next_insn->type != INSN_TRAP) { 4780 WARN_INSN(insn, "missing int3 after ret"); 4781 warnings++; 4782 } 4783 4784 break; 4785 case INSN_JUMP_DYNAMIC: 4786 if (!next_insn || next_insn->type != INSN_TRAP) { 4787 WARN_INSN(insn, "missing int3 after indirect jump"); 4788 warnings++; 4789 } 4790 break; 4791 default: 4792 break; 4793 } 4794 } 4795 4796 return warnings; 4797 } 4798 4799 static int validate_reachable_instructions(struct objtool_file *file) 4800 { 4801 struct instruction *insn, *prev_insn; 4802 struct symbol *call_dest; 4803 int warnings = 0; 4804 4805 if (file->ignore_unreachables) 4806 return 0; 4807 4808 for_each_insn(file, insn) { 4809 if (insn->visited || ignore_unreachable_insn(file, insn)) 4810 continue; 4811 4812 prev_insn = prev_insn_same_sec(file, insn); 4813 if (prev_insn && prev_insn->dead_end) { 4814 call_dest = insn_call_dest(prev_insn); 4815 if (call_dest) { 4816 WARN_INSN(insn, "%s() missing __noreturn in .c/.h or NORETURN() in noreturns.h", 4817 call_dest->name); 4818 warnings++; 4819 continue; 4820 } 4821 } 4822 4823 WARN_INSN(insn, "unreachable instruction"); 4824 warnings++; 4825 } 4826 4827 return warnings; 4828 } 4829 4830 __weak bool arch_absolute_reloc(struct elf *elf, struct reloc *reloc) 4831 { 4832 unsigned int type = reloc_type(reloc); 4833 size_t sz = elf_addr_size(elf); 4834 4835 return (sz == 8) ? (type == R_ABS64) : (type == R_ABS32); 4836 } 4837 4838 static int check_abs_references(struct objtool_file *file) 4839 { 4840 struct section *sec; 4841 struct reloc *reloc; 4842 int ret = 0; 4843 4844 for_each_sec(file->elf, sec) { 4845 /* absolute references in non-loadable sections are fine */ 4846 if (!(sec->sh.sh_flags & SHF_ALLOC)) 4847 continue; 4848 4849 /* section must have an associated .rela section */ 4850 if (!sec->rsec) 4851 continue; 4852 4853 /* 4854 * Special case for compiler generated metadata that is not 4855 * consumed until after boot. 4856 */ 4857 if (!strcmp(sec->name, "__patchable_function_entries")) 4858 continue; 4859 4860 for_each_reloc(sec->rsec, reloc) { 4861 if (arch_absolute_reloc(file->elf, reloc)) { 4862 WARN("section %s has absolute relocation at offset 0x%llx", 4863 sec->name, (unsigned long long)reloc_offset(reloc)); 4864 ret++; 4865 } 4866 } 4867 } 4868 return ret; 4869 } 4870 4871 struct insn_chunk { 4872 void *addr; 4873 struct insn_chunk *next; 4874 }; 4875 4876 /* 4877 * Reduce peak RSS usage by freeing insns memory before writing the ELF file, 4878 * which can trigger more allocations for .debug_* sections whose data hasn't 4879 * been read yet. 4880 */ 4881 static void free_insns(struct objtool_file *file) 4882 { 4883 struct instruction *insn; 4884 struct insn_chunk *chunks = NULL, *chunk; 4885 4886 for_each_insn(file, insn) { 4887 if (!insn->idx) { 4888 chunk = malloc(sizeof(*chunk)); 4889 chunk->addr = insn; 4890 chunk->next = chunks; 4891 chunks = chunk; 4892 } 4893 } 4894 4895 for (chunk = chunks; chunk; chunk = chunk->next) 4896 free(chunk->addr); 4897 } 4898 4899 const char *objtool_disas_insn(struct instruction *insn) 4900 { 4901 struct disas_context *dctx = objtool_disas_ctx; 4902 4903 if (!dctx) 4904 return ""; 4905 4906 disas_insn(dctx, insn); 4907 return disas_result(dctx); 4908 } 4909 4910 int check(struct objtool_file *file) 4911 { 4912 struct disas_context *disas_ctx = NULL; 4913 int ret = 0, warnings = 0; 4914 4915 /* 4916 * If the verbose or backtrace option is used then we need a 4917 * disassembly context to disassemble instruction or function 4918 * on warning or backtrace. 4919 */ 4920 if (opts.verbose || opts.backtrace || opts.trace) { 4921 disas_ctx = disas_context_create(file); 4922 if (!disas_ctx) 4923 opts.trace = false; 4924 objtool_disas_ctx = disas_ctx; 4925 } 4926 4927 arch_initial_func_cfi_state(&initial_func_cfi); 4928 init_cfi_state(&init_cfi); 4929 init_cfi_state(&func_cfi); 4930 set_func_state(&func_cfi); 4931 init_cfi_state(&force_undefined_cfi); 4932 force_undefined_cfi.force_undefined = true; 4933 4934 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) { 4935 ret = -1; 4936 goto out; 4937 } 4938 4939 cfi_hash_add(&init_cfi); 4940 cfi_hash_add(&func_cfi); 4941 4942 ret = checksum_debug_init(file); 4943 if (ret) 4944 goto out; 4945 4946 ret = decode_sections(file); 4947 if (ret) 4948 goto out; 4949 4950 if (!nr_insns) 4951 goto out; 4952 4953 if (opts.retpoline) 4954 warnings += validate_retpoline(file); 4955 4956 if (validate_branch_enabled()) { 4957 int w = 0; 4958 4959 w += validate_functions(file); 4960 w += validate_unwind_hints(file, NULL); 4961 if (!w) 4962 w += validate_reachable_instructions(file); 4963 4964 warnings += w; 4965 4966 } else if (opts.noinstr) { 4967 warnings += validate_noinstr_sections(file); 4968 } 4969 4970 if (opts.unret) { 4971 /* 4972 * Must be after validate_branch() and friends, it plays 4973 * further games with insn->visited. 4974 */ 4975 warnings += validate_unrets(file); 4976 } 4977 4978 if (opts.ibt) 4979 warnings += validate_ibt(file); 4980 4981 if (opts.sls) 4982 warnings += validate_sls(file); 4983 4984 if (opts.static_call) { 4985 ret = create_static_call_sections(file); 4986 if (ret) 4987 goto out; 4988 } 4989 4990 if (opts.retpoline) { 4991 ret = create_retpoline_sites_sections(file); 4992 if (ret) 4993 goto out; 4994 } 4995 4996 if (opts.cfi) { 4997 ret = create_cfi_sections(file); 4998 if (ret) 4999 goto out; 5000 } 5001 5002 if (opts.rethunk) { 5003 ret = create_return_sites_sections(file); 5004 if (ret) 5005 goto out; 5006 5007 if (opts.hack_skylake) { 5008 ret = create_direct_call_sections(file); 5009 if (ret) 5010 goto out; 5011 } 5012 } 5013 5014 if (opts.mcount) { 5015 ret = create_mcount_loc_sections(file); 5016 if (ret) 5017 goto out; 5018 } 5019 5020 if (opts.prefix) { 5021 ret = create_prefix_symbols(file); 5022 if (ret) 5023 goto out; 5024 } 5025 5026 if (opts.ibt) { 5027 ret = create_ibt_endbr_seal_sections(file); 5028 if (ret) 5029 goto out; 5030 } 5031 5032 if (opts.noabs) 5033 warnings += check_abs_references(file); 5034 5035 if (opts.checksum) { 5036 ret = create_sym_checksum_section(file); 5037 if (ret) 5038 goto out; 5039 } 5040 5041 if (opts.orc && nr_insns) { 5042 ret = orc_create(file); 5043 if (ret) 5044 goto out; 5045 } 5046 5047 if (opts.stats) { 5048 printf("nr_insns_visited: %ld\n", nr_insns_visited); 5049 printf("nr_cfi: %ld\n", nr_cfi); 5050 printf("nr_cfi_reused: %ld\n", nr_cfi_reused); 5051 printf("nr_cfi_cache: %ld\n", nr_cfi_cache); 5052 } 5053 5054 out: 5055 if (!ret && !warnings) { 5056 free_insns(file); 5057 return 0; 5058 } 5059 5060 if (opts.werror && warnings) 5061 ret = 1; 5062 5063 if (opts.verbose) { 5064 if (opts.werror && warnings) 5065 WARN("%d warning(s) upgraded to errors", warnings); 5066 disas_warned_funcs(disas_ctx); 5067 } 5068 5069 if (disas_ctx) { 5070 disas_context_destroy(disas_ctx); 5071 objtool_disas_ctx = NULL; 5072 } 5073 5074 free_insns(file); 5075 5076 if (opts.backup && make_backup()) 5077 return 1; 5078 5079 return ret; 5080 } 5081