1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6 #define _GNU_SOURCE /* memmem() */ 7 #include <fnmatch.h> 8 #include <string.h> 9 #include <stdlib.h> 10 #include <inttypes.h> 11 #include <sys/mman.h> 12 13 #include <objtool/builtin.h> 14 #include <objtool/cfi.h> 15 #include <objtool/arch.h> 16 #include <objtool/disas.h> 17 #include <objtool/check.h> 18 #include <objtool/special.h> 19 #include <objtool/trace.h> 20 #include <objtool/warn.h> 21 #include <objtool/checksum.h> 22 #include <objtool/util.h> 23 24 #include <linux/objtool_types.h> 25 #include <linux/hashtable.h> 26 #include <linux/kernel.h> 27 #include <linux/static_call_types.h> 28 #include <linux/string.h> 29 30 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 31 32 static struct cfi_init_state initial_func_cfi; 33 static struct cfi_state init_cfi; 34 static struct cfi_state func_cfi; 35 static struct cfi_state force_undefined_cfi; 36 37 struct disas_context *objtool_disas_ctx; 38 39 size_t sym_name_max_len; 40 41 struct instruction *find_insn(struct objtool_file *file, 42 struct section *sec, unsigned long offset) 43 { 44 struct instruction *insn; 45 46 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) { 47 if (insn->sec == sec && insn->offset == offset) 48 return insn; 49 } 50 51 return NULL; 52 } 53 54 struct instruction *next_insn_same_sec(struct objtool_file *file, 55 struct instruction *insn) 56 { 57 if (insn->idx == INSN_CHUNK_MAX) 58 return find_insn(file, insn->sec, insn->offset + insn->len); 59 60 insn++; 61 if (!insn->len) 62 return NULL; 63 64 return insn; 65 } 66 67 static struct instruction *next_insn_same_func(struct objtool_file *file, 68 struct instruction *insn) 69 { 70 struct instruction *next = next_insn_same_sec(file, insn); 71 struct symbol *func = insn_func(insn); 72 73 if (!func) 74 return NULL; 75 76 if (next && insn_func(next) == func) 77 return next; 78 79 /* Check if we're already in the subfunction: */ 80 if (func == func->cfunc) 81 return NULL; 82 83 /* Move to the subfunction: */ 84 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 85 } 86 87 static struct instruction *prev_insn_same_sec(struct objtool_file *file, 88 struct instruction *insn) 89 { 90 if (insn->idx == 0) { 91 if (insn->prev_len) 92 return find_insn(file, insn->sec, insn->offset - insn->prev_len); 93 return NULL; 94 } 95 96 return insn - 1; 97 } 98 99 static struct instruction *prev_insn_same_sym(struct objtool_file *file, 100 struct instruction *insn) 101 { 102 struct instruction *prev = prev_insn_same_sec(file, insn); 103 104 if (prev && insn_func(prev) == insn_func(insn)) 105 return prev; 106 107 return NULL; 108 } 109 110 #define for_each_insn(file, insn) \ 111 for (struct section *__sec, *__fake = (struct section *)1; \ 112 __fake; __fake = NULL) \ 113 for_each_sec(file->elf, __sec) \ 114 sec_for_each_insn(file, __sec, insn) 115 116 #define func_for_each_insn(file, func, insn) \ 117 for (insn = find_insn(file, func->sec, func->offset); \ 118 insn; \ 119 insn = next_insn_same_func(file, insn)) 120 121 #define sym_for_each_insn(file, sym, insn) \ 122 for (insn = find_insn(file, sym->sec, sym->offset); \ 123 insn && insn->offset < sym->offset + sym->len; \ 124 insn = next_insn_same_sec(file, insn)) 125 126 #define sym_for_each_insn_continue_reverse(file, sym, insn) \ 127 for (insn = prev_insn_same_sec(file, insn); \ 128 insn && insn->offset >= sym->offset; \ 129 insn = prev_insn_same_sec(file, insn)) 130 131 #define sec_for_each_insn_from(file, insn) \ 132 for (; insn; insn = next_insn_same_sec(file, insn)) 133 134 #define sec_for_each_insn_continue(file, insn) \ 135 for (insn = next_insn_same_sec(file, insn); insn; \ 136 insn = next_insn_same_sec(file, insn)) 137 138 static inline struct reloc *insn_jump_table(struct instruction *insn) 139 { 140 if (insn->type == INSN_JUMP_DYNAMIC || 141 insn->type == INSN_CALL_DYNAMIC) 142 return insn->_jump_table; 143 144 return NULL; 145 } 146 147 static inline unsigned long insn_jump_table_size(struct instruction *insn) 148 { 149 if (insn->type == INSN_JUMP_DYNAMIC || 150 insn->type == INSN_CALL_DYNAMIC) 151 return insn->_jump_table_size; 152 153 return 0; 154 } 155 156 static bool is_jump_table_jump(struct instruction *insn) 157 { 158 struct alt_group *alt_group = insn->alt_group; 159 160 if (insn_jump_table(insn)) 161 return true; 162 163 /* Retpoline alternative for a jump table? */ 164 return alt_group && alt_group->orig_group && 165 insn_jump_table(alt_group->orig_group->first_insn); 166 } 167 168 static bool is_sibling_call(struct instruction *insn) 169 { 170 /* 171 * Assume only STT_FUNC calls have jump-tables. 172 */ 173 if (insn_func(insn)) { 174 /* An indirect jump is either a sibling call or a jump to a table. */ 175 if (insn->type == INSN_JUMP_DYNAMIC) 176 return !is_jump_table_jump(insn); 177 } 178 179 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */ 180 return (is_static_jump(insn) && insn_call_dest(insn)); 181 } 182 183 /* 184 * Checks if a function is a Rust "noreturn" one. 185 */ 186 static bool is_rust_noreturn(const struct symbol *func) 187 { 188 /* 189 * If it does not start with "_R", then it is not a Rust symbol. 190 */ 191 if (strncmp(func->name, "_R", 2)) 192 return false; 193 194 /* 195 * These are just heuristics -- we do not control the precise symbol 196 * name, due to the crate disambiguators (which depend on the compiler) 197 * as well as changes to the source code itself between versions (since 198 * these come from the Rust standard library). 199 */ 200 return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || 201 str_ends_with(func->name, "_4core6option13expect_failed") || 202 str_ends_with(func->name, "_4core6option13unwrap_failed") || 203 str_ends_with(func->name, "_4core6result13unwrap_failed") || 204 str_ends_with(func->name, "_4core9panicking5panic") || 205 str_ends_with(func->name, "_4core9panicking9panic_fmt") || 206 str_ends_with(func->name, "_4core9panicking14panic_explicit") || 207 str_ends_with(func->name, "_4core9panicking14panic_nounwind") || 208 str_ends_with(func->name, "_4core9panicking18panic_bounds_check") || 209 str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt") || 210 str_ends_with(func->name, "_4core9panicking19assert_failed_inner") || 211 str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference") || 212 str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") || 213 str_ends_with(func->name, "_7___rustc17rust_begin_unwind") || 214 strstr(func->name, "_4core9panicking13assert_failed") || 215 strstr(func->name, "_4core9panicking11panic_const24panic_const_") || 216 (strstr(func->name, "_4core5slice5index") && 217 strstr(func->name, "slice_") && 218 str_ends_with(func->name, "_fail")); 219 } 220 221 /* 222 * This checks to see if the given function is a "noreturn" function. 223 * 224 * For global functions which are outside the scope of this object file, we 225 * have to keep a manual list of them. 226 * 227 * For local functions, we have to detect them manually by simply looking for 228 * the lack of a return instruction. 229 */ 230 static bool __dead_end_function(struct objtool_file *file, struct symbol *func, 231 int recursion) 232 { 233 int i; 234 struct instruction *insn; 235 bool empty = true; 236 237 #define NORETURN(func) __stringify(func), 238 static const char * const global_noreturns[] = { 239 #include "noreturns.h" 240 }; 241 #undef NORETURN 242 243 if (!func) 244 return false; 245 246 if (!is_local_sym(func)) { 247 if (is_rust_noreturn(func)) 248 return true; 249 250 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) 251 if (!strcmp(func->name, global_noreturns[i])) 252 return true; 253 } 254 255 if (is_weak_sym(func)) 256 return false; 257 258 if (!func->len) 259 return false; 260 261 insn = find_insn(file, func->sec, func->offset); 262 if (!insn || !insn_func(insn)) 263 return false; 264 265 func_for_each_insn(file, func, insn) { 266 empty = false; 267 268 if (insn->type == INSN_RETURN) 269 return false; 270 } 271 272 if (empty) 273 return false; 274 275 /* 276 * A function can have a sibling call instead of a return. In that 277 * case, the function's dead-end status depends on whether the target 278 * of the sibling call returns. 279 */ 280 func_for_each_insn(file, func, insn) { 281 if (is_sibling_call(insn)) { 282 struct instruction *dest = insn->jump_dest; 283 284 if (!dest) 285 /* sibling call to another file */ 286 return false; 287 288 /* local sibling call */ 289 if (recursion == 5) { 290 /* 291 * Infinite recursion: two functions have 292 * sibling calls to each other. This is a very 293 * rare case. It means they aren't dead ends. 294 */ 295 return false; 296 } 297 298 return __dead_end_function(file, insn_func(dest), recursion+1); 299 } 300 } 301 302 return true; 303 } 304 305 static bool dead_end_function(struct objtool_file *file, struct symbol *func) 306 { 307 return __dead_end_function(file, func, 0); 308 } 309 310 static void init_cfi_state(struct cfi_state *cfi) 311 { 312 int i; 313 314 for (i = 0; i < CFI_NUM_REGS; i++) { 315 cfi->regs[i].base = CFI_UNDEFINED; 316 cfi->vals[i].base = CFI_UNDEFINED; 317 } 318 cfi->cfa.base = CFI_UNDEFINED; 319 cfi->drap_reg = CFI_UNDEFINED; 320 cfi->drap_offset = -1; 321 } 322 323 static void init_insn_state(struct objtool_file *file, struct insn_state *state, 324 struct section *sec) 325 { 326 memset(state, 0, sizeof(*state)); 327 init_cfi_state(&state->cfi); 328 329 if (opts.noinstr && sec) 330 state->noinstr = sec->noinstr; 331 } 332 333 static struct cfi_state *cfi_alloc(void) 334 { 335 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state)); 336 if (!cfi) { 337 ERROR_GLIBC("calloc"); 338 exit(1); 339 } 340 nr_cfi++; 341 return cfi; 342 } 343 344 static int cfi_bits; 345 static struct hlist_head *cfi_hash; 346 347 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2) 348 { 349 return memcmp((void *)cfi1 + sizeof(cfi1->hash), 350 (void *)cfi2 + sizeof(cfi2->hash), 351 sizeof(struct cfi_state) - sizeof(struct hlist_node)); 352 } 353 354 static inline u32 cfi_key(struct cfi_state *cfi) 355 { 356 return jhash((void *)cfi + sizeof(cfi->hash), 357 sizeof(*cfi) - sizeof(cfi->hash), 0); 358 } 359 360 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi) 361 { 362 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 363 struct cfi_state *obj; 364 365 hlist_for_each_entry(obj, head, hash) { 366 if (!cficmp(cfi, obj)) { 367 nr_cfi_cache++; 368 return obj; 369 } 370 } 371 372 obj = cfi_alloc(); 373 *obj = *cfi; 374 hlist_add_head(&obj->hash, head); 375 376 return obj; 377 } 378 379 static void cfi_hash_add(struct cfi_state *cfi) 380 { 381 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 382 383 hlist_add_head(&cfi->hash, head); 384 } 385 386 static void *cfi_hash_alloc(unsigned long size) 387 { 388 cfi_bits = max(10, ilog2(size)); 389 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits, 390 PROT_READ|PROT_WRITE, 391 MAP_PRIVATE|MAP_ANON, -1, 0); 392 if (cfi_hash == (void *)-1L) { 393 ERROR_GLIBC("mmap fail cfi_hash"); 394 cfi_hash = NULL; 395 } else if (opts.stats) { 396 printf("cfi_bits: %d\n", cfi_bits); 397 } 398 399 return cfi_hash; 400 } 401 402 static unsigned long nr_insns; 403 static unsigned long nr_insns_visited; 404 405 /* 406 * Call the arch-specific instruction decoder for all the instructions and add 407 * them to the global instruction list. 408 */ 409 static int decode_instructions(struct objtool_file *file) 410 { 411 struct section *sec; 412 struct symbol *func; 413 unsigned long offset; 414 struct instruction *insn; 415 416 for_each_sec(file->elf, sec) { 417 struct instruction *insns = NULL; 418 u8 prev_len = 0; 419 u8 idx = 0; 420 421 if (!is_text_sec(sec)) 422 continue; 423 424 if (strcmp(sec->name, ".altinstr_replacement") && 425 strcmp(sec->name, ".altinstr_aux") && 426 strncmp(sec->name, ".discard.", 9)) 427 sec->text = true; 428 429 if (!strcmp(sec->name, ".noinstr.text") || 430 !strcmp(sec->name, ".entry.text") || 431 !strcmp(sec->name, ".cpuidle.text") || 432 !strncmp(sec->name, ".text..__x86.", 13)) 433 sec->noinstr = true; 434 435 /* 436 * .init.text code is ran before userspace and thus doesn't 437 * strictly need retpolines, except for modules which are 438 * loaded late, they very much do need retpoline in their 439 * .init.text 440 */ 441 if (!strcmp(sec->name, ".init.text") && !opts.module) 442 sec->init = true; 443 444 for (offset = 0; offset < sec_size(sec); offset += insn->len) { 445 if (!insns || idx == INSN_CHUNK_MAX) { 446 insns = calloc(INSN_CHUNK_SIZE, sizeof(*insn)); 447 if (!insns) { 448 ERROR_GLIBC("calloc"); 449 return -1; 450 } 451 idx = 0; 452 } else { 453 idx++; 454 } 455 insn = &insns[idx]; 456 insn->idx = idx; 457 458 INIT_LIST_HEAD(&insn->call_node); 459 insn->sec = sec; 460 insn->offset = offset; 461 insn->prev_len = prev_len; 462 463 if (arch_decode_instruction(file, sec, offset, sec_size(sec) - offset, insn)) 464 return -1; 465 466 prev_len = insn->len; 467 468 /* 469 * By default, "ud2" is a dead end unless otherwise 470 * annotated, because GCC 7 inserts it for certain 471 * divide-by-zero cases. 472 */ 473 if (insn->type == INSN_BUG) 474 insn->dead_end = true; 475 476 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset)); 477 nr_insns++; 478 } 479 480 sec_for_each_sym(sec, func) { 481 if (!is_notype_sym(func) && !is_func_sym(func)) 482 continue; 483 484 if (func->offset == sec_size(sec)) { 485 /* Heuristic: likely an "end" symbol */ 486 if (is_notype_sym(func)) 487 continue; 488 ERROR("%s(): STT_FUNC at end of section", func->name); 489 return -1; 490 } 491 492 if (func->embedded_insn || func->alias != func) 493 continue; 494 495 if (!find_insn(file, sec, func->offset)) { 496 ERROR("%s(): can't find starting instruction", func->name); 497 return -1; 498 } 499 500 sym_for_each_insn(file, func, insn) { 501 insn->sym = func; 502 if (is_func_sym(func) && 503 insn->type == INSN_ENDBR && 504 list_empty(&insn->call_node)) { 505 if (insn->offset == func->offset) { 506 list_add_tail(&insn->call_node, &file->endbr_list); 507 file->nr_endbr++; 508 } else { 509 file->nr_endbr_int++; 510 } 511 } 512 } 513 } 514 } 515 516 if (opts.stats) 517 printf("nr_insns: %lu\n", nr_insns); 518 519 return 0; 520 } 521 522 /* 523 * Read the pv_ops[] .data table to find the static initialized values. 524 */ 525 static int add_pv_ops(struct objtool_file *file, const char *symname) 526 { 527 struct symbol *sym, *func; 528 unsigned long off, end; 529 struct reloc *reloc; 530 int idx; 531 532 sym = find_symbol_by_name(file->elf, symname); 533 if (!sym) 534 return 0; 535 536 off = sym->offset; 537 end = off + sym->len; 538 for (;;) { 539 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 540 if (!reloc) 541 break; 542 543 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long); 544 545 func = reloc->sym; 546 if (is_sec_sym(func)) 547 func = find_symbol_by_offset(reloc->sym->sec, 548 reloc_addend(reloc)); 549 if (!func) { 550 ERROR_FUNC(reloc->sym->sec, reloc_addend(reloc), 551 "can't find func at %s[%d]", symname, idx); 552 return -1; 553 } 554 555 if (objtool_pv_add(file, idx, func)) 556 return -1; 557 558 off = reloc_offset(reloc) + 1; 559 if (off > end) 560 break; 561 } 562 563 return 0; 564 } 565 566 /* 567 * Allocate and initialize file->pv_ops[]. 568 */ 569 static int init_pv_ops(struct objtool_file *file) 570 { 571 static const char *pv_ops_tables[] = { 572 "pv_ops", 573 "xen_cpu_ops", 574 "xen_irq_ops", 575 "xen_mmu_ops", 576 NULL, 577 }; 578 const char *pv_ops; 579 struct symbol *sym; 580 int idx, nr; 581 582 if (!opts.noinstr) 583 return 0; 584 585 file->pv_ops = NULL; 586 587 sym = find_symbol_by_name(file->elf, "pv_ops"); 588 if (!sym) 589 return 0; 590 591 nr = sym->len / sizeof(unsigned long); 592 file->pv_ops = calloc(nr, sizeof(struct pv_state)); 593 if (!file->pv_ops) { 594 ERROR_GLIBC("calloc"); 595 return -1; 596 } 597 598 for (idx = 0; idx < nr; idx++) 599 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 600 601 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) { 602 if (add_pv_ops(file, pv_ops)) 603 return -1; 604 } 605 606 return 0; 607 } 608 609 static bool is_livepatch_module(struct objtool_file *file) 610 { 611 struct section *sec; 612 613 if (!opts.module) 614 return false; 615 616 sec = find_section_by_name(file->elf, ".modinfo"); 617 if (!sec) 618 return false; 619 620 return memmem(sec->data->d_buf, sec_size(sec), "\0livepatch=Y", 12); 621 } 622 623 static int create_static_call_sections(struct objtool_file *file) 624 { 625 struct static_call_site *site; 626 struct section *sec; 627 struct instruction *insn; 628 struct symbol *key_sym; 629 char *key_name, *tmp; 630 int idx; 631 632 sec = find_section_by_name(file->elf, ".static_call_sites"); 633 if (sec) { 634 /* 635 * Livepatch modules may have already extracted the static call 636 * site entries to take advantage of vmlinux static call 637 * privileges. 638 */ 639 if (!file->klp) 640 WARN("file already has .static_call_sites section, skipping"); 641 642 return 0; 643 } 644 645 if (list_empty(&file->static_call_list)) 646 return 0; 647 648 idx = 0; 649 list_for_each_entry(insn, &file->static_call_list, call_node) 650 idx++; 651 652 sec = elf_create_section_pair(file->elf, ".static_call_sites", 653 sizeof(*site), idx, idx * 2); 654 if (!sec) 655 return -1; 656 657 /* Allow modules to modify the low bits of static_call_site::key */ 658 sec->sh.sh_flags |= SHF_WRITE; 659 660 idx = 0; 661 list_for_each_entry(insn, &file->static_call_list, call_node) { 662 663 /* populate reloc for 'addr' */ 664 if (!elf_init_reloc_text_sym(file->elf, sec, 665 idx * sizeof(*site), idx * 2, 666 insn->sec, insn->offset)) 667 return -1; 668 669 /* find key symbol */ 670 key_name = strdup(insn_call_dest(insn)->name); 671 if (!key_name) { 672 ERROR_GLIBC("strdup"); 673 return -1; 674 } 675 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR, 676 STATIC_CALL_TRAMP_PREFIX_LEN)) { 677 ERROR("static_call: trampoline name malformed: %s", key_name); 678 return -1; 679 } 680 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN; 681 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN); 682 683 key_sym = find_symbol_by_name(file->elf, tmp); 684 if (!key_sym) { 685 if (!opts.module || file->klp) { 686 ERROR("static_call: can't find static_call_key symbol: %s", tmp); 687 return -1; 688 } 689 690 /* 691 * For modules(), the key might not be exported, which 692 * means the module can make static calls but isn't 693 * allowed to change them. 694 * 695 * In that case we temporarily set the key to be the 696 * trampoline address. This is fixed up in 697 * static_call_add_module(). 698 */ 699 key_sym = insn_call_dest(insn); 700 } 701 702 /* populate reloc for 'key' */ 703 if (!elf_init_reloc_data_sym(file->elf, sec, 704 idx * sizeof(*site) + 4, 705 (idx * 2) + 1, key_sym, 706 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL)) 707 return -1; 708 709 idx++; 710 } 711 712 return 0; 713 } 714 715 static int create_retpoline_sites_sections(struct objtool_file *file) 716 { 717 struct instruction *insn; 718 struct section *sec; 719 int idx; 720 721 sec = find_section_by_name(file->elf, ".retpoline_sites"); 722 if (sec) { 723 WARN("file already has .retpoline_sites, skipping"); 724 return 0; 725 } 726 727 idx = 0; 728 list_for_each_entry(insn, &file->retpoline_call_list, call_node) 729 idx++; 730 731 if (!idx) 732 return 0; 733 734 sec = elf_create_section_pair(file->elf, ".retpoline_sites", 735 sizeof(int), idx, idx); 736 if (!sec) 737 return -1; 738 739 idx = 0; 740 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 741 742 if (!elf_init_reloc_text_sym(file->elf, sec, 743 idx * sizeof(int), idx, 744 insn->sec, insn->offset)) 745 return -1; 746 747 idx++; 748 } 749 750 return 0; 751 } 752 753 static int create_return_sites_sections(struct objtool_file *file) 754 { 755 struct instruction *insn; 756 struct section *sec; 757 int idx; 758 759 sec = find_section_by_name(file->elf, ".return_sites"); 760 if (sec) { 761 WARN("file already has .return_sites, skipping"); 762 return 0; 763 } 764 765 idx = 0; 766 list_for_each_entry(insn, &file->return_thunk_list, call_node) 767 idx++; 768 769 if (!idx) 770 return 0; 771 772 sec = elf_create_section_pair(file->elf, ".return_sites", 773 sizeof(int), idx, idx); 774 if (!sec) 775 return -1; 776 777 idx = 0; 778 list_for_each_entry(insn, &file->return_thunk_list, call_node) { 779 780 if (!elf_init_reloc_text_sym(file->elf, sec, 781 idx * sizeof(int), idx, 782 insn->sec, insn->offset)) 783 return -1; 784 785 idx++; 786 } 787 788 return 0; 789 } 790 791 static int create_ibt_endbr_seal_sections(struct objtool_file *file) 792 { 793 struct instruction *insn; 794 struct section *sec; 795 int idx; 796 797 sec = find_section_by_name(file->elf, ".ibt_endbr_seal"); 798 if (sec) { 799 WARN("file already has .ibt_endbr_seal, skipping"); 800 return 0; 801 } 802 803 idx = 0; 804 list_for_each_entry(insn, &file->endbr_list, call_node) 805 idx++; 806 807 if (opts.stats) { 808 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); 809 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); 810 printf("ibt: superfluous ENDBR: %d\n", idx); 811 } 812 813 if (!idx) 814 return 0; 815 816 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal", 817 sizeof(int), idx, idx); 818 if (!sec) 819 return -1; 820 821 idx = 0; 822 list_for_each_entry(insn, &file->endbr_list, call_node) { 823 824 int *site = (int *)sec->data->d_buf + idx; 825 struct symbol *sym = insn->sym; 826 *site = 0; 827 828 if (opts.module && sym && is_func_sym(sym) && 829 insn->offset == sym->offset && 830 (!strcmp(sym->name, "init_module") || 831 !strcmp(sym->name, "cleanup_module"))) { 832 ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead", 833 sym->name); 834 return -1; 835 } 836 837 if (!elf_init_reloc_text_sym(file->elf, sec, 838 idx * sizeof(int), idx, 839 insn->sec, insn->offset)) 840 return -1; 841 842 idx++; 843 } 844 845 return 0; 846 } 847 848 static int create_cfi_sections(struct objtool_file *file) 849 { 850 struct section *sec; 851 struct symbol *sym; 852 int idx; 853 854 sec = find_section_by_name(file->elf, ".cfi_sites"); 855 if (sec) { 856 WARN("file already has .cfi_sites section, skipping"); 857 return 0; 858 } 859 860 idx = 0; 861 for_each_sym(file->elf, sym) { 862 if (!is_func_sym(sym)) 863 continue; 864 865 if (strncmp(sym->name, "__cfi_", 6)) 866 continue; 867 868 idx++; 869 } 870 871 sec = elf_create_section_pair(file->elf, ".cfi_sites", 872 sizeof(unsigned int), idx, idx); 873 if (!sec) 874 return -1; 875 876 idx = 0; 877 for_each_sym(file->elf, sym) { 878 if (!is_func_sym(sym)) 879 continue; 880 881 if (strncmp(sym->name, "__cfi_", 6)) 882 continue; 883 884 if (!elf_init_reloc_text_sym(file->elf, sec, 885 idx * sizeof(unsigned int), idx, 886 sym->sec, sym->offset)) 887 return -1; 888 889 idx++; 890 } 891 892 return 0; 893 } 894 895 static int create_mcount_loc_sections(struct objtool_file *file) 896 { 897 size_t addr_size = elf_addr_size(file->elf); 898 struct instruction *insn; 899 struct section *sec; 900 int idx; 901 902 sec = find_section_by_name(file->elf, "__mcount_loc"); 903 if (sec) { 904 /* 905 * Livepatch modules have already extracted their __mcount_loc 906 * entries to cover the !CONFIG_FTRACE_MCOUNT_USE_OBJTOOL case. 907 */ 908 if (!file->klp) 909 WARN("file already has __mcount_loc section, skipping"); 910 911 return 0; 912 } 913 914 if (list_empty(&file->mcount_loc_list)) 915 return 0; 916 917 idx = 0; 918 list_for_each_entry(insn, &file->mcount_loc_list, call_node) 919 idx++; 920 921 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size, 922 idx, idx); 923 if (!sec) 924 return -1; 925 926 sec->sh.sh_addralign = addr_size; 927 928 idx = 0; 929 list_for_each_entry(insn, &file->mcount_loc_list, call_node) { 930 931 struct reloc *reloc; 932 933 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx, 934 insn->sec, insn->offset); 935 if (!reloc) 936 return -1; 937 938 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32); 939 940 idx++; 941 } 942 943 return 0; 944 } 945 946 static int create_direct_call_sections(struct objtool_file *file) 947 { 948 struct instruction *insn; 949 struct section *sec; 950 int idx; 951 952 sec = find_section_by_name(file->elf, ".call_sites"); 953 if (sec) { 954 WARN("file already has .call_sites section, skipping"); 955 return 0; 956 } 957 958 if (list_empty(&file->call_list)) 959 return 0; 960 961 idx = 0; 962 list_for_each_entry(insn, &file->call_list, call_node) 963 idx++; 964 965 sec = elf_create_section_pair(file->elf, ".call_sites", 966 sizeof(unsigned int), idx, idx); 967 if (!sec) 968 return -1; 969 970 idx = 0; 971 list_for_each_entry(insn, &file->call_list, call_node) { 972 973 if (!elf_init_reloc_text_sym(file->elf, sec, 974 idx * sizeof(unsigned int), idx, 975 insn->sec, insn->offset)) 976 return -1; 977 978 idx++; 979 } 980 981 return 0; 982 } 983 984 #ifdef BUILD_KLP 985 static int create_sym_checksum_section(struct objtool_file *file) 986 { 987 struct section *sec; 988 struct symbol *sym; 989 unsigned int idx = 0; 990 struct sym_checksum *checksum; 991 size_t entsize = sizeof(struct sym_checksum); 992 993 sec = find_section_by_name(file->elf, ".discard.sym_checksum"); 994 if (sec) { 995 if (!opts.dryrun) 996 WARN("file already has .discard.sym_checksum section, skipping"); 997 998 return 0; 999 } 1000 1001 for_each_sym(file->elf, sym) 1002 if (sym->csum.checksum) 1003 idx++; 1004 1005 if (!idx) 1006 return 0; 1007 1008 sec = elf_create_section_pair(file->elf, ".discard.sym_checksum", entsize, 1009 idx, idx); 1010 if (!sec) 1011 return -1; 1012 1013 idx = 0; 1014 for_each_sym(file->elf, sym) { 1015 if (!sym->csum.checksum) 1016 continue; 1017 1018 if (!elf_init_reloc(file->elf, sec->rsec, idx, idx * entsize, 1019 sym, 0, R_TEXT64)) 1020 return -1; 1021 1022 checksum = (struct sym_checksum *)sec->data->d_buf + idx; 1023 checksum->addr = 0; /* reloc */ 1024 checksum->checksum = sym->csum.checksum; 1025 1026 mark_sec_changed(file->elf, sec, true); 1027 1028 idx++; 1029 } 1030 1031 return 0; 1032 } 1033 #else 1034 static int create_sym_checksum_section(struct objtool_file *file) { return -EINVAL; } 1035 #endif 1036 1037 /* 1038 * Warnings shouldn't be reported for ignored functions. 1039 */ 1040 static int add_ignores(struct objtool_file *file) 1041 { 1042 struct section *rsec; 1043 struct symbol *func; 1044 struct reloc *reloc; 1045 1046 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard"); 1047 if (!rsec) 1048 return 0; 1049 1050 for_each_reloc(rsec, reloc) { 1051 switch (reloc->sym->type) { 1052 case STT_FUNC: 1053 func = reloc->sym; 1054 break; 1055 1056 case STT_SECTION: 1057 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc)); 1058 if (!func) 1059 continue; 1060 break; 1061 1062 default: 1063 ERROR("unexpected relocation symbol type in %s: %d", 1064 rsec->name, reloc->sym->type); 1065 return -1; 1066 } 1067 1068 func->ignore = true; 1069 if (func->cfunc) 1070 func->cfunc->ignore = true; 1071 } 1072 1073 return 0; 1074 } 1075 1076 /* 1077 * This is a whitelist of functions that is allowed to be called with AC set. 1078 * The list is meant to be minimal and only contains compiler instrumentation 1079 * ABI and a few functions used to implement *_{to,from}_user() functions. 1080 * 1081 * These functions must not directly change AC, but may PUSHF/POPF. 1082 */ 1083 static const char *uaccess_safe_builtin[] = { 1084 /* KASAN */ 1085 "kasan_report", 1086 "kasan_check_range", 1087 /* KASAN out-of-line */ 1088 "__asan_loadN_noabort", 1089 "__asan_load1_noabort", 1090 "__asan_load2_noabort", 1091 "__asan_load4_noabort", 1092 "__asan_load8_noabort", 1093 "__asan_load16_noabort", 1094 "__asan_storeN_noabort", 1095 "__asan_store1_noabort", 1096 "__asan_store2_noabort", 1097 "__asan_store4_noabort", 1098 "__asan_store8_noabort", 1099 "__asan_store16_noabort", 1100 "__kasan_check_read", 1101 "__kasan_check_write", 1102 /* KASAN in-line */ 1103 "__asan_report_load_n_noabort", 1104 "__asan_report_load1_noabort", 1105 "__asan_report_load2_noabort", 1106 "__asan_report_load4_noabort", 1107 "__asan_report_load8_noabort", 1108 "__asan_report_load16_noabort", 1109 "__asan_report_store_n_noabort", 1110 "__asan_report_store1_noabort", 1111 "__asan_report_store2_noabort", 1112 "__asan_report_store4_noabort", 1113 "__asan_report_store8_noabort", 1114 "__asan_report_store16_noabort", 1115 /* KCSAN */ 1116 "__kcsan_check_access", 1117 "__kcsan_mb", 1118 "__kcsan_wmb", 1119 "__kcsan_rmb", 1120 "__kcsan_release", 1121 "kcsan_found_watchpoint", 1122 "kcsan_setup_watchpoint", 1123 "kcsan_check_scoped_accesses", 1124 "kcsan_disable_current", 1125 "kcsan_enable_current_nowarn", 1126 /* KCSAN/TSAN */ 1127 "__tsan_func_entry", 1128 "__tsan_func_exit", 1129 "__tsan_read_range", 1130 "__tsan_write_range", 1131 "__tsan_read1", 1132 "__tsan_read2", 1133 "__tsan_read4", 1134 "__tsan_read8", 1135 "__tsan_read16", 1136 "__tsan_write1", 1137 "__tsan_write2", 1138 "__tsan_write4", 1139 "__tsan_write8", 1140 "__tsan_write16", 1141 "__tsan_read_write1", 1142 "__tsan_read_write2", 1143 "__tsan_read_write4", 1144 "__tsan_read_write8", 1145 "__tsan_read_write16", 1146 "__tsan_volatile_read1", 1147 "__tsan_volatile_read2", 1148 "__tsan_volatile_read4", 1149 "__tsan_volatile_read8", 1150 "__tsan_volatile_read16", 1151 "__tsan_volatile_write1", 1152 "__tsan_volatile_write2", 1153 "__tsan_volatile_write4", 1154 "__tsan_volatile_write8", 1155 "__tsan_volatile_write16", 1156 "__tsan_atomic8_load", 1157 "__tsan_atomic16_load", 1158 "__tsan_atomic32_load", 1159 "__tsan_atomic64_load", 1160 "__tsan_atomic8_store", 1161 "__tsan_atomic16_store", 1162 "__tsan_atomic32_store", 1163 "__tsan_atomic64_store", 1164 "__tsan_atomic8_exchange", 1165 "__tsan_atomic16_exchange", 1166 "__tsan_atomic32_exchange", 1167 "__tsan_atomic64_exchange", 1168 "__tsan_atomic8_fetch_add", 1169 "__tsan_atomic16_fetch_add", 1170 "__tsan_atomic32_fetch_add", 1171 "__tsan_atomic64_fetch_add", 1172 "__tsan_atomic8_fetch_sub", 1173 "__tsan_atomic16_fetch_sub", 1174 "__tsan_atomic32_fetch_sub", 1175 "__tsan_atomic64_fetch_sub", 1176 "__tsan_atomic8_fetch_and", 1177 "__tsan_atomic16_fetch_and", 1178 "__tsan_atomic32_fetch_and", 1179 "__tsan_atomic64_fetch_and", 1180 "__tsan_atomic8_fetch_or", 1181 "__tsan_atomic16_fetch_or", 1182 "__tsan_atomic32_fetch_or", 1183 "__tsan_atomic64_fetch_or", 1184 "__tsan_atomic8_fetch_xor", 1185 "__tsan_atomic16_fetch_xor", 1186 "__tsan_atomic32_fetch_xor", 1187 "__tsan_atomic64_fetch_xor", 1188 "__tsan_atomic8_fetch_nand", 1189 "__tsan_atomic16_fetch_nand", 1190 "__tsan_atomic32_fetch_nand", 1191 "__tsan_atomic64_fetch_nand", 1192 "__tsan_atomic8_compare_exchange_strong", 1193 "__tsan_atomic16_compare_exchange_strong", 1194 "__tsan_atomic32_compare_exchange_strong", 1195 "__tsan_atomic64_compare_exchange_strong", 1196 "__tsan_atomic8_compare_exchange_weak", 1197 "__tsan_atomic16_compare_exchange_weak", 1198 "__tsan_atomic32_compare_exchange_weak", 1199 "__tsan_atomic64_compare_exchange_weak", 1200 "__tsan_atomic8_compare_exchange_val", 1201 "__tsan_atomic16_compare_exchange_val", 1202 "__tsan_atomic32_compare_exchange_val", 1203 "__tsan_atomic64_compare_exchange_val", 1204 "__tsan_atomic_thread_fence", 1205 "__tsan_atomic_signal_fence", 1206 "__tsan_unaligned_read16", 1207 "__tsan_unaligned_write16", 1208 /* KCOV */ 1209 "write_comp_data", 1210 "check_kcov_mode", 1211 "__sanitizer_cov_trace_pc", 1212 "__sanitizer_cov_trace_const_cmp1", 1213 "__sanitizer_cov_trace_const_cmp2", 1214 "__sanitizer_cov_trace_const_cmp4", 1215 "__sanitizer_cov_trace_const_cmp8", 1216 "__sanitizer_cov_trace_cmp1", 1217 "__sanitizer_cov_trace_cmp2", 1218 "__sanitizer_cov_trace_cmp4", 1219 "__sanitizer_cov_trace_cmp8", 1220 "__sanitizer_cov_trace_switch", 1221 /* KMSAN */ 1222 "kmsan_copy_to_user", 1223 "kmsan_disable_current", 1224 "kmsan_enable_current", 1225 "kmsan_report", 1226 "kmsan_unpoison_entry_regs", 1227 "kmsan_unpoison_memory", 1228 "__msan_chain_origin", 1229 "__msan_get_context_state", 1230 "__msan_instrument_asm_store", 1231 "__msan_metadata_ptr_for_load_1", 1232 "__msan_metadata_ptr_for_load_2", 1233 "__msan_metadata_ptr_for_load_4", 1234 "__msan_metadata_ptr_for_load_8", 1235 "__msan_metadata_ptr_for_load_n", 1236 "__msan_metadata_ptr_for_store_1", 1237 "__msan_metadata_ptr_for_store_2", 1238 "__msan_metadata_ptr_for_store_4", 1239 "__msan_metadata_ptr_for_store_8", 1240 "__msan_metadata_ptr_for_store_n", 1241 "__msan_poison_alloca", 1242 "__msan_warning", 1243 /* UBSAN */ 1244 "ubsan_type_mismatch_common", 1245 "__ubsan_handle_type_mismatch", 1246 "__ubsan_handle_type_mismatch_v1", 1247 "__ubsan_handle_shift_out_of_bounds", 1248 "__ubsan_handle_load_invalid_value", 1249 /* KSTACK_ERASE */ 1250 "__sanitizer_cov_stack_depth", 1251 /* TRACE_BRANCH_PROFILING */ 1252 "ftrace_likely_update", 1253 /* STACKPROTECTOR */ 1254 "__stack_chk_fail", 1255 /* misc */ 1256 "csum_partial_copy_generic", 1257 "copy_mc_fragile", 1258 "copy_mc_fragile_handle_tail", 1259 "copy_mc_enhanced_fast_string", 1260 "rep_stos_alternative", 1261 "rep_movs_alternative", 1262 "__copy_user_nocache", 1263 NULL 1264 }; 1265 1266 static void add_uaccess_safe(struct objtool_file *file) 1267 { 1268 struct symbol *func; 1269 const char **name; 1270 1271 if (!opts.uaccess) 1272 return; 1273 1274 for (name = uaccess_safe_builtin; *name; name++) { 1275 func = find_symbol_by_name(file->elf, *name); 1276 if (!func) 1277 continue; 1278 1279 func->uaccess_safe = true; 1280 } 1281 } 1282 1283 /* 1284 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol 1285 * will be added to the .retpoline_sites section. 1286 */ 1287 __weak bool arch_is_retpoline(struct symbol *sym) 1288 { 1289 return false; 1290 } 1291 1292 /* 1293 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol 1294 * will be added to the .return_sites section. 1295 */ 1296 __weak bool arch_is_rethunk(struct symbol *sym) 1297 { 1298 return false; 1299 } 1300 1301 /* 1302 * Symbols that are embedded inside other instructions, because sometimes crazy 1303 * code exists. These are mostly ignored for validation purposes. 1304 */ 1305 __weak bool arch_is_embedded_insn(struct symbol *sym) 1306 { 1307 return false; 1308 } 1309 1310 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) 1311 { 1312 struct reloc *reloc; 1313 1314 if (insn->no_reloc) 1315 return NULL; 1316 1317 if (!file) 1318 return NULL; 1319 1320 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 1321 insn->offset, insn->len); 1322 if (!reloc) { 1323 insn->no_reloc = 1; 1324 return NULL; 1325 } 1326 1327 return reloc; 1328 } 1329 1330 static void remove_insn_ops(struct instruction *insn) 1331 { 1332 struct stack_op *op, *next; 1333 1334 for (op = insn->stack_ops; op; op = next) { 1335 next = op->next; 1336 free(op); 1337 } 1338 insn->stack_ops = NULL; 1339 } 1340 1341 static int annotate_call_site(struct objtool_file *file, 1342 struct instruction *insn, bool sibling) 1343 { 1344 struct reloc *reloc = insn_reloc(file, insn); 1345 struct symbol *sym = insn_call_dest(insn); 1346 1347 if (!sym) 1348 sym = reloc->sym; 1349 1350 if (sym->static_call_tramp) { 1351 list_add_tail(&insn->call_node, &file->static_call_list); 1352 return 0; 1353 } 1354 1355 if (sym->retpoline_thunk) { 1356 list_add_tail(&insn->call_node, &file->retpoline_call_list); 1357 return 0; 1358 } 1359 1360 /* 1361 * Many compilers cannot disable KCOV or sanitizer calls with a function 1362 * attribute so they need a little help, NOP out any such calls from 1363 * noinstr text. 1364 */ 1365 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) { 1366 if (reloc) 1367 set_reloc_type(file->elf, reloc, R_NONE); 1368 1369 if (elf_write_insn(file->elf, insn->sec, 1370 insn->offset, insn->len, 1371 sibling ? arch_ret_insn(insn->len) 1372 : arch_nop_insn(insn->len))) { 1373 return -1; 1374 } 1375 1376 insn->type = sibling ? INSN_RETURN : INSN_NOP; 1377 1378 if (sibling) { 1379 /* 1380 * We've replaced the tail-call JMP insn by two new 1381 * insn: RET; INT3, except we only have a single struct 1382 * insn here. Mark it retpoline_safe to avoid the SLS 1383 * warning, instead of adding another insn. 1384 */ 1385 insn->retpoline_safe = true; 1386 } 1387 1388 return 0; 1389 } 1390 1391 if (opts.mcount && sym->fentry) { 1392 if (sibling) 1393 WARN_INSN(insn, "tail call to __fentry__ !?!?"); 1394 if (opts.mnop) { 1395 if (reloc) 1396 set_reloc_type(file->elf, reloc, R_NONE); 1397 1398 if (elf_write_insn(file->elf, insn->sec, 1399 insn->offset, insn->len, 1400 arch_nop_insn(insn->len))) { 1401 return -1; 1402 } 1403 1404 insn->type = INSN_NOP; 1405 } 1406 1407 list_add_tail(&insn->call_node, &file->mcount_loc_list); 1408 return 0; 1409 } 1410 1411 if (insn->type == INSN_CALL && !insn->sec->init && 1412 !insn->_call_dest->embedded_insn) 1413 list_add_tail(&insn->call_node, &file->call_list); 1414 1415 if (!sibling && dead_end_function(file, sym)) 1416 insn->dead_end = true; 1417 1418 return 0; 1419 } 1420 1421 static int add_call_dest(struct objtool_file *file, struct instruction *insn, 1422 struct symbol *dest, bool sibling) 1423 { 1424 insn->_call_dest = dest; 1425 if (!dest) 1426 return 0; 1427 1428 /* 1429 * Whatever stack impact regular CALLs have, should be undone 1430 * by the RETURN of the called function. 1431 * 1432 * Annotated intra-function calls retain the stack_ops but 1433 * are converted to JUMP, see read_intra_function_calls(). 1434 */ 1435 remove_insn_ops(insn); 1436 1437 return annotate_call_site(file, insn, sibling); 1438 } 1439 1440 static int add_retpoline_call(struct objtool_file *file, struct instruction *insn) 1441 { 1442 /* 1443 * Retpoline calls/jumps are really dynamic calls/jumps in disguise, 1444 * so convert them accordingly. 1445 */ 1446 switch (insn->type) { 1447 case INSN_CALL: 1448 insn->type = INSN_CALL_DYNAMIC; 1449 break; 1450 case INSN_JUMP_UNCONDITIONAL: 1451 insn->type = INSN_JUMP_DYNAMIC; 1452 break; 1453 case INSN_JUMP_CONDITIONAL: 1454 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL; 1455 break; 1456 default: 1457 return 0; 1458 } 1459 1460 insn->retpoline_safe = true; 1461 1462 /* 1463 * Whatever stack impact regular CALLs have, should be undone 1464 * by the RETURN of the called function. 1465 * 1466 * Annotated intra-function calls retain the stack_ops but 1467 * are converted to JUMP, see read_intra_function_calls(). 1468 */ 1469 remove_insn_ops(insn); 1470 1471 return annotate_call_site(file, insn, false); 1472 } 1473 1474 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) 1475 { 1476 /* 1477 * Return thunk tail calls are really just returns in disguise, 1478 * so convert them accordingly. 1479 */ 1480 insn->type = INSN_RETURN; 1481 insn->retpoline_safe = true; 1482 1483 if (add) 1484 list_add_tail(&insn->call_node, &file->return_thunk_list); 1485 } 1486 1487 static bool is_first_func_insn(struct objtool_file *file, 1488 struct instruction *insn) 1489 { 1490 struct symbol *func = insn_func(insn); 1491 1492 if (!func) 1493 return false; 1494 1495 if (insn->offset == func->offset) 1496 return true; 1497 1498 /* Allow direct CALL/JMP past ENDBR */ 1499 if (opts.ibt) { 1500 struct instruction *prev = prev_insn_same_sym(file, insn); 1501 1502 if (prev && prev->type == INSN_ENDBR && 1503 insn->offset == func->offset + prev->len) 1504 return true; 1505 } 1506 1507 return false; 1508 } 1509 1510 /* 1511 * Find the destination instructions for all jumps. 1512 */ 1513 static int add_jump_destinations(struct objtool_file *file) 1514 { 1515 struct instruction *insn; 1516 struct reloc *reloc; 1517 1518 for_each_insn(file, insn) { 1519 struct symbol *func = insn_func(insn); 1520 struct instruction *dest_insn; 1521 struct section *dest_sec; 1522 struct symbol *dest_sym; 1523 unsigned long dest_off; 1524 1525 if (!is_static_jump(insn)) 1526 continue; 1527 1528 if (insn->jump_dest) { 1529 /* 1530 * handle_group_alt() may have previously set 1531 * 'jump_dest' for some alternatives. 1532 */ 1533 continue; 1534 } 1535 1536 reloc = insn_reloc(file, insn); 1537 if (!reloc) { 1538 dest_sec = insn->sec; 1539 dest_off = arch_jump_destination(insn); 1540 dest_sym = dest_sec->sym; 1541 } else { 1542 dest_sym = reloc->sym; 1543 if (is_undef_sym(dest_sym)) { 1544 if (dest_sym->retpoline_thunk) { 1545 if (add_retpoline_call(file, insn)) 1546 return -1; 1547 continue; 1548 } 1549 1550 if (dest_sym->return_thunk) { 1551 add_return_call(file, insn, true); 1552 continue; 1553 } 1554 1555 /* External symbol */ 1556 if (func) { 1557 /* External sibling call */ 1558 if (add_call_dest(file, insn, dest_sym, true)) 1559 return -1; 1560 continue; 1561 } 1562 1563 /* Non-func asm code jumping to external symbol */ 1564 continue; 1565 } 1566 1567 dest_sec = dest_sym->sec; 1568 dest_off = dest_sym->offset + arch_insn_adjusted_addend(insn, reloc); 1569 } 1570 1571 dest_insn = find_insn(file, dest_sec, dest_off); 1572 if (!dest_insn) { 1573 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); 1574 1575 /* 1576 * retbleed_untrain_ret() jumps to 1577 * __x86_return_thunk(), but objtool can't find 1578 * the thunk's starting RET instruction, 1579 * because the RET is also in the middle of 1580 * another instruction. Objtool only knows 1581 * about the outer instruction. 1582 */ 1583 if (sym && sym->embedded_insn) { 1584 add_return_call(file, insn, false); 1585 continue; 1586 } 1587 1588 /* 1589 * GCOV/KCOV dead code can jump to the end of 1590 * the function/section. 1591 */ 1592 if (file->ignore_unreachables && func && 1593 dest_sec == insn->sec && 1594 dest_off == func->offset + func->len) 1595 continue; 1596 1597 ERROR_INSN(insn, "can't find jump dest instruction at %s", 1598 offstr(dest_sec, dest_off)); 1599 return -1; 1600 } 1601 1602 if (!dest_sym || is_sec_sym(dest_sym)) { 1603 dest_sym = dest_insn->sym; 1604 if (!dest_sym) 1605 goto set_jump_dest; 1606 } 1607 1608 if (dest_sym->retpoline_thunk && dest_insn->offset == dest_sym->offset) { 1609 if (add_retpoline_call(file, insn)) 1610 return -1; 1611 continue; 1612 } 1613 1614 if (dest_sym->return_thunk && dest_insn->offset == dest_sym->offset) { 1615 add_return_call(file, insn, true); 1616 continue; 1617 } 1618 1619 if (!insn->sym || insn->sym->pfunc == dest_sym->pfunc) 1620 goto set_jump_dest; 1621 1622 /* 1623 * Internal cross-function jump. 1624 */ 1625 1626 if (is_first_func_insn(file, dest_insn)) { 1627 /* Internal sibling call */ 1628 if (add_call_dest(file, insn, dest_sym, true)) 1629 return -1; 1630 continue; 1631 } 1632 1633 set_jump_dest: 1634 insn->jump_dest = dest_insn; 1635 } 1636 1637 return 0; 1638 } 1639 1640 static struct symbol *find_call_destination(struct section *sec, unsigned long offset) 1641 { 1642 struct symbol *call_dest; 1643 1644 call_dest = find_func_by_offset(sec, offset); 1645 if (!call_dest) 1646 call_dest = find_symbol_by_offset(sec, offset); 1647 1648 return call_dest; 1649 } 1650 1651 /* 1652 * Find the destination instructions for all calls. 1653 */ 1654 static int add_call_destinations(struct objtool_file *file) 1655 { 1656 struct instruction *insn; 1657 unsigned long dest_off; 1658 struct symbol *dest; 1659 struct reloc *reloc; 1660 1661 for_each_insn(file, insn) { 1662 struct symbol *func = insn_func(insn); 1663 if (insn->type != INSN_CALL) 1664 continue; 1665 1666 reloc = insn_reloc(file, insn); 1667 if (!reloc) { 1668 dest_off = arch_jump_destination(insn); 1669 dest = find_call_destination(insn->sec, dest_off); 1670 1671 if (add_call_dest(file, insn, dest, false)) 1672 return -1; 1673 1674 if (func && func->ignore) 1675 continue; 1676 1677 if (!insn_call_dest(insn)) { 1678 ERROR_INSN(insn, "unannotated intra-function call"); 1679 return -1; 1680 } 1681 1682 if (func && !is_func_sym(insn_call_dest(insn))) { 1683 ERROR_INSN(insn, "unsupported call to non-function"); 1684 return -1; 1685 } 1686 1687 } else if (is_sec_sym(reloc->sym)) { 1688 dest_off = arch_insn_adjusted_addend(insn, reloc); 1689 dest = find_call_destination(reloc->sym->sec, dest_off); 1690 if (!dest) { 1691 ERROR_INSN(insn, "can't find call dest symbol at %s+0x%lx", 1692 reloc->sym->sec->name, dest_off); 1693 return -1; 1694 } 1695 1696 if (add_call_dest(file, insn, dest, false)) 1697 return -1; 1698 1699 } else if (reloc->sym->retpoline_thunk) { 1700 if (add_retpoline_call(file, insn)) 1701 return -1; 1702 1703 } else { 1704 if (add_call_dest(file, insn, reloc->sym, false)) 1705 return -1; 1706 } 1707 } 1708 1709 return 0; 1710 } 1711 1712 /* 1713 * The .alternatives section requires some extra special care over and above 1714 * other special sections because alternatives are patched in place. 1715 */ 1716 static int handle_group_alt(struct objtool_file *file, 1717 struct special_alt *special_alt, 1718 struct instruction *orig_insn, 1719 struct instruction **new_insn) 1720 { 1721 struct instruction *last_new_insn = NULL, *insn, *nop = NULL; 1722 struct alt_group *orig_alt_group, *new_alt_group; 1723 unsigned long dest_off; 1724 1725 orig_alt_group = orig_insn->alt_group; 1726 if (!orig_alt_group) { 1727 struct instruction *last_orig_insn = NULL; 1728 1729 orig_alt_group = calloc(1, sizeof(*orig_alt_group)); 1730 if (!orig_alt_group) { 1731 ERROR_GLIBC("calloc"); 1732 return -1; 1733 } 1734 orig_alt_group->cfi = calloc(special_alt->orig_len, 1735 sizeof(struct cfi_state *)); 1736 if (!orig_alt_group->cfi) { 1737 ERROR_GLIBC("calloc"); 1738 return -1; 1739 } 1740 1741 insn = orig_insn; 1742 sec_for_each_insn_from(file, insn) { 1743 if (insn->offset >= special_alt->orig_off + special_alt->orig_len) 1744 break; 1745 1746 insn->alt_group = orig_alt_group; 1747 last_orig_insn = insn; 1748 } 1749 orig_alt_group->orig_group = NULL; 1750 orig_alt_group->first_insn = orig_insn; 1751 orig_alt_group->last_insn = last_orig_insn; 1752 orig_alt_group->nop = NULL; 1753 orig_alt_group->ignore = orig_insn->ignore_alts; 1754 } else { 1755 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - 1756 orig_alt_group->first_insn->offset != special_alt->orig_len) { 1757 ERROR_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d", 1758 orig_alt_group->last_insn->offset + 1759 orig_alt_group->last_insn->len - 1760 orig_alt_group->first_insn->offset, 1761 special_alt->orig_len); 1762 return -1; 1763 } 1764 } 1765 1766 new_alt_group = calloc(1, sizeof(*new_alt_group)); 1767 if (!new_alt_group) { 1768 ERROR_GLIBC("calloc"); 1769 return -1; 1770 } 1771 1772 if (special_alt->new_len < special_alt->orig_len) { 1773 /* 1774 * Insert a fake nop at the end to make the replacement 1775 * alt_group the same size as the original. This is needed to 1776 * allow propagate_alt_cfi() to do its magic. When the last 1777 * instruction affects the stack, the instruction after it (the 1778 * nop) will propagate the new state to the shared CFI array. 1779 */ 1780 nop = calloc(1, sizeof(*nop)); 1781 if (!nop) { 1782 ERROR_GLIBC("calloc"); 1783 return -1; 1784 } 1785 memset(nop, 0, sizeof(*nop)); 1786 1787 nop->sec = special_alt->new_sec; 1788 nop->offset = special_alt->new_off + special_alt->new_len; 1789 nop->len = special_alt->orig_len - special_alt->new_len; 1790 nop->type = INSN_NOP; 1791 nop->sym = orig_insn->sym; 1792 nop->alt_group = new_alt_group; 1793 nop->fake = 1; 1794 } 1795 1796 if (!special_alt->new_len) { 1797 *new_insn = nop; 1798 goto end; 1799 } 1800 1801 insn = *new_insn; 1802 sec_for_each_insn_from(file, insn) { 1803 struct reloc *alt_reloc; 1804 1805 if (insn->offset >= special_alt->new_off + special_alt->new_len) 1806 break; 1807 1808 last_new_insn = insn; 1809 1810 insn->sym = orig_insn->sym; 1811 insn->alt_group = new_alt_group; 1812 1813 /* 1814 * Since alternative replacement code is copy/pasted by the 1815 * kernel after applying relocations, generally such code can't 1816 * have relative-address relocation references to outside the 1817 * .altinstr_replacement section, unless the arch's 1818 * alternatives code can adjust the relative offsets 1819 * accordingly. 1820 */ 1821 alt_reloc = insn_reloc(file, insn); 1822 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) && 1823 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { 1824 1825 ERROR_INSN(insn, "unsupported relocation in alternatives section"); 1826 return -1; 1827 } 1828 1829 if (!is_static_jump(insn)) 1830 continue; 1831 1832 if (!insn->immediate) 1833 continue; 1834 1835 dest_off = arch_jump_destination(insn); 1836 if (dest_off == special_alt->new_off + special_alt->new_len) { 1837 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn); 1838 if (!insn->jump_dest) { 1839 ERROR_INSN(insn, "can't find alternative jump destination"); 1840 return -1; 1841 } 1842 } 1843 } 1844 1845 if (!last_new_insn) { 1846 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1847 "can't find last new alternative instruction"); 1848 return -1; 1849 } 1850 1851 end: 1852 new_alt_group->orig_group = orig_alt_group; 1853 new_alt_group->first_insn = *new_insn; 1854 new_alt_group->last_insn = last_new_insn; 1855 new_alt_group->nop = nop; 1856 new_alt_group->ignore = (*new_insn)->ignore_alts; 1857 new_alt_group->cfi = orig_alt_group->cfi; 1858 return 0; 1859 } 1860 1861 /* 1862 * A jump table entry can either convert a nop to a jump or a jump to a nop. 1863 * If the original instruction is a jump, make the alt entry an effective nop 1864 * by just skipping the original instruction. 1865 */ 1866 static int handle_jump_alt(struct objtool_file *file, 1867 struct special_alt *special_alt, 1868 struct instruction *orig_insn, 1869 struct instruction **new_insn) 1870 { 1871 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && 1872 orig_insn->type != INSN_NOP) { 1873 1874 ERROR_INSN(orig_insn, "unsupported instruction at jump label"); 1875 return -1; 1876 } 1877 1878 if (opts.hack_jump_label && special_alt->key_addend & 2) { 1879 struct reloc *reloc = insn_reloc(file, orig_insn); 1880 1881 if (reloc) 1882 set_reloc_type(file->elf, reloc, R_NONE); 1883 1884 if (elf_write_insn(file->elf, orig_insn->sec, 1885 orig_insn->offset, orig_insn->len, 1886 arch_nop_insn(orig_insn->len))) { 1887 return -1; 1888 } 1889 1890 orig_insn->type = INSN_NOP; 1891 } 1892 1893 if (orig_insn->type == INSN_NOP) { 1894 if (orig_insn->len == 2) 1895 file->jl_nop_short++; 1896 else 1897 file->jl_nop_long++; 1898 1899 return 0; 1900 } 1901 1902 if (orig_insn->len == 2) 1903 file->jl_short++; 1904 else 1905 file->jl_long++; 1906 1907 *new_insn = next_insn_same_sec(file, orig_insn); 1908 return 0; 1909 } 1910 1911 /* 1912 * Read all the special sections which have alternate instructions which can be 1913 * patched in or redirected to at runtime. Each instruction having alternate 1914 * instruction(s) has them added to its insn->alts list, which will be 1915 * traversed in validate_branch(). 1916 */ 1917 static int add_special_section_alts(struct objtool_file *file) 1918 { 1919 struct list_head special_alts; 1920 struct instruction *orig_insn, *new_insn; 1921 struct special_alt *special_alt, *tmp; 1922 enum alternative_type alt_type; 1923 struct alternative *alt; 1924 1925 if (special_get_alts(file->elf, &special_alts)) 1926 return -1; 1927 1928 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) { 1929 1930 orig_insn = find_insn(file, special_alt->orig_sec, 1931 special_alt->orig_off); 1932 if (!orig_insn) { 1933 ERROR_FUNC(special_alt->orig_sec, special_alt->orig_off, 1934 "special: can't find orig instruction"); 1935 return -1; 1936 } 1937 1938 new_insn = NULL; 1939 if (!special_alt->group || special_alt->new_len) { 1940 new_insn = find_insn(file, special_alt->new_sec, 1941 special_alt->new_off); 1942 if (!new_insn) { 1943 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1944 "special: can't find new instruction"); 1945 return -1; 1946 } 1947 } 1948 1949 if (special_alt->group) { 1950 if (!special_alt->orig_len) { 1951 ERROR_INSN(orig_insn, "empty alternative entry"); 1952 continue; 1953 } 1954 1955 if (handle_group_alt(file, special_alt, orig_insn, &new_insn)) 1956 return -1; 1957 1958 alt_type = ALT_TYPE_INSTRUCTIONS; 1959 1960 } else if (special_alt->jump_or_nop) { 1961 if (handle_jump_alt(file, special_alt, orig_insn, &new_insn)) 1962 return -1; 1963 1964 alt_type = ALT_TYPE_JUMP_TABLE; 1965 } else { 1966 alt_type = ALT_TYPE_EX_TABLE; 1967 } 1968 1969 alt = calloc(1, sizeof(*alt)); 1970 if (!alt) { 1971 ERROR_GLIBC("calloc"); 1972 return -1; 1973 } 1974 1975 alt->insn = new_insn; 1976 alt->next = orig_insn->alts; 1977 alt->type = alt_type; 1978 orig_insn->alts = alt; 1979 1980 list_del(&special_alt->list); 1981 free(special_alt); 1982 } 1983 1984 if (opts.stats) { 1985 printf("jl\\\tNOP\tJMP\n"); 1986 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); 1987 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); 1988 } 1989 1990 return 0; 1991 } 1992 1993 __weak unsigned long arch_jump_table_sym_offset(struct reloc *reloc, struct reloc *table) 1994 { 1995 return reloc->sym->offset + reloc_addend(reloc); 1996 } 1997 1998 static int add_jump_table(struct objtool_file *file, struct instruction *insn) 1999 { 2000 unsigned long table_size = insn_jump_table_size(insn); 2001 struct symbol *pfunc = insn_func(insn)->pfunc; 2002 struct reloc *table = insn_jump_table(insn); 2003 struct instruction *dest_insn; 2004 unsigned int prev_offset = 0; 2005 struct reloc *reloc = table; 2006 struct alternative *alt; 2007 unsigned long sym_offset; 2008 2009 /* 2010 * Each @reloc is a switch table relocation which points to the target 2011 * instruction. 2012 */ 2013 for_each_reloc_from(table->sec, reloc) { 2014 2015 /* Check for the end of the table: */ 2016 if (table_size && reloc_offset(reloc) - reloc_offset(table) >= table_size) 2017 break; 2018 if (reloc != table && is_jump_table(reloc)) 2019 break; 2020 2021 /* Make sure the table entries are consecutive: */ 2022 if (prev_offset && reloc_offset(reloc) != prev_offset + arch_reloc_size(reloc)) 2023 break; 2024 2025 sym_offset = arch_jump_table_sym_offset(reloc, table); 2026 2027 /* Detect function pointers from contiguous objects: */ 2028 if (reloc->sym->sec == pfunc->sec && sym_offset == pfunc->offset) 2029 break; 2030 2031 /* 2032 * Clang sometimes leaves dangling unused jump table entries 2033 * which point to the end of the function. Ignore them. 2034 */ 2035 if (reloc->sym->sec == pfunc->sec && 2036 sym_offset == pfunc->offset + pfunc->len) 2037 goto next; 2038 2039 dest_insn = find_insn(file, reloc->sym->sec, sym_offset); 2040 if (!dest_insn) 2041 break; 2042 2043 /* Make sure the destination is in the same function: */ 2044 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc) 2045 break; 2046 2047 alt = calloc(1, sizeof(*alt)); 2048 if (!alt) { 2049 ERROR_GLIBC("calloc"); 2050 return -1; 2051 } 2052 2053 alt->insn = dest_insn; 2054 alt->next = insn->alts; 2055 insn->alts = alt; 2056 next: 2057 prev_offset = reloc_offset(reloc); 2058 } 2059 2060 if (!prev_offset) { 2061 ERROR_INSN(insn, "can't find switch jump table"); 2062 return -1; 2063 } 2064 2065 return 0; 2066 } 2067 2068 /* 2069 * find_jump_table() - Given a dynamic jump, find the switch jump table 2070 * associated with it. 2071 */ 2072 static void find_jump_table(struct objtool_file *file, struct symbol *func, 2073 struct instruction *insn) 2074 { 2075 struct reloc *table_reloc; 2076 struct instruction *dest_insn, *orig_insn = insn; 2077 unsigned long table_size; 2078 unsigned long sym_offset; 2079 2080 /* 2081 * Backward search using the @first_jump_src links, these help avoid 2082 * much of the 'in between' code. Which avoids us getting confused by 2083 * it. 2084 */ 2085 for (; 2086 insn && insn_func(insn) && insn_func(insn)->pfunc == func; 2087 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) { 2088 2089 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC) 2090 break; 2091 2092 /* allow small jumps within the range */ 2093 if (insn->type == INSN_JUMP_UNCONDITIONAL && 2094 insn->jump_dest && 2095 (insn->jump_dest->offset <= insn->offset || 2096 insn->jump_dest->offset > orig_insn->offset)) 2097 break; 2098 2099 table_reloc = arch_find_switch_table(file, insn, &table_size); 2100 if (!table_reloc) 2101 continue; 2102 2103 sym_offset = table_reloc->sym->offset + reloc_addend(table_reloc); 2104 2105 dest_insn = find_insn(file, table_reloc->sym->sec, sym_offset); 2106 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func) 2107 continue; 2108 2109 set_jump_table(table_reloc); 2110 orig_insn->_jump_table = table_reloc; 2111 orig_insn->_jump_table_size = table_size; 2112 2113 break; 2114 } 2115 } 2116 2117 /* 2118 * First pass: Mark the head of each jump table so that in the next pass, 2119 * we know when a given jump table ends and the next one starts. 2120 */ 2121 static void mark_func_jump_tables(struct objtool_file *file, 2122 struct symbol *func) 2123 { 2124 struct instruction *insn, *last = NULL; 2125 2126 func_for_each_insn(file, func, insn) { 2127 if (!last) 2128 last = insn; 2129 2130 /* 2131 * Store back-pointers for unconditional forward jumps such 2132 * that find_jump_table() can back-track using those and 2133 * avoid some potentially confusing code. 2134 */ 2135 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && 2136 insn->offset > last->offset && 2137 insn->jump_dest->offset > insn->offset && 2138 !insn->jump_dest->first_jump_src) { 2139 2140 insn->jump_dest->first_jump_src = insn; 2141 last = insn->jump_dest; 2142 } 2143 2144 if (insn->type != INSN_JUMP_DYNAMIC) 2145 continue; 2146 2147 find_jump_table(file, func, insn); 2148 } 2149 } 2150 2151 static int add_func_jump_tables(struct objtool_file *file, 2152 struct symbol *func) 2153 { 2154 struct instruction *insn; 2155 2156 func_for_each_insn(file, func, insn) { 2157 if (!insn_jump_table(insn)) 2158 continue; 2159 2160 if (add_jump_table(file, insn)) 2161 return -1; 2162 } 2163 2164 return 0; 2165 } 2166 2167 /* 2168 * For some switch statements, gcc generates a jump table in the .rodata 2169 * section which contains a list of addresses within the function to jump to. 2170 * This finds these jump tables and adds them to the insn->alts lists. 2171 */ 2172 static int add_jump_table_alts(struct objtool_file *file) 2173 { 2174 struct symbol *func; 2175 2176 if (!file->rodata) 2177 return 0; 2178 2179 for_each_sym(file->elf, func) { 2180 if (!is_func_sym(func) || func->alias != func) 2181 continue; 2182 2183 mark_func_jump_tables(file, func); 2184 if (add_func_jump_tables(file, func)) 2185 return -1; 2186 } 2187 2188 return 0; 2189 } 2190 2191 static void set_func_state(struct cfi_state *state) 2192 { 2193 state->cfa = initial_func_cfi.cfa; 2194 memcpy(&state->regs, &initial_func_cfi.regs, 2195 CFI_NUM_REGS * sizeof(struct cfi_reg)); 2196 state->stack_size = initial_func_cfi.cfa.offset; 2197 state->type = UNWIND_HINT_TYPE_CALL; 2198 } 2199 2200 static int read_unwind_hints(struct objtool_file *file) 2201 { 2202 struct cfi_state cfi = init_cfi; 2203 struct section *sec; 2204 struct unwind_hint *hint; 2205 struct instruction *insn; 2206 struct reloc *reloc; 2207 unsigned long offset; 2208 int i; 2209 2210 sec = find_section_by_name(file->elf, ".discard.unwind_hints"); 2211 if (!sec) 2212 return 0; 2213 2214 if (!sec->rsec) { 2215 ERROR("missing .rela.discard.unwind_hints section"); 2216 return -1; 2217 } 2218 2219 if (sec_size(sec) % sizeof(struct unwind_hint)) { 2220 ERROR("struct unwind_hint size mismatch"); 2221 return -1; 2222 } 2223 2224 file->hints = true; 2225 2226 for (i = 0; i < sec_size(sec) / sizeof(struct unwind_hint); i++) { 2227 hint = (struct unwind_hint *)sec->data->d_buf + i; 2228 2229 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); 2230 if (!reloc) { 2231 ERROR("can't find reloc for unwind_hints[%d]", i); 2232 return -1; 2233 } 2234 2235 offset = reloc->sym->offset + reloc_addend(reloc); 2236 2237 insn = find_insn(file, reloc->sym->sec, offset); 2238 if (!insn) { 2239 ERROR("can't find insn for unwind_hints[%d]", i); 2240 return -1; 2241 } 2242 2243 insn->hint = true; 2244 2245 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) { 2246 insn->cfi = &force_undefined_cfi; 2247 continue; 2248 } 2249 2250 if (hint->type == UNWIND_HINT_TYPE_SAVE) { 2251 insn->hint = false; 2252 insn->save = true; 2253 continue; 2254 } 2255 2256 if (hint->type == UNWIND_HINT_TYPE_RESTORE) { 2257 insn->restore = true; 2258 continue; 2259 } 2260 2261 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 2262 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 2263 2264 if (sym && is_global_sym(sym)) { 2265 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { 2266 ERROR_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR"); 2267 return -1; 2268 } 2269 } 2270 } 2271 2272 if (hint->type == UNWIND_HINT_TYPE_FUNC) { 2273 insn->cfi = &func_cfi; 2274 continue; 2275 } 2276 2277 if (insn->cfi) 2278 cfi = *(insn->cfi); 2279 2280 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { 2281 ERROR_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg); 2282 return -1; 2283 } 2284 2285 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); 2286 cfi.type = hint->type; 2287 cfi.signal = hint->signal; 2288 2289 insn->cfi = cfi_hash_find_or_add(&cfi); 2290 } 2291 2292 return 0; 2293 } 2294 2295 static int read_annotate(struct objtool_file *file, 2296 int (*func)(struct objtool_file *file, int type, struct instruction *insn)) 2297 { 2298 struct section *sec; 2299 struct instruction *insn; 2300 struct reloc *reloc; 2301 uint64_t offset; 2302 int type; 2303 2304 sec = find_section_by_name(file->elf, ".discard.annotate_insn"); 2305 if (!sec) 2306 return 0; 2307 2308 if (!sec->rsec) 2309 return 0; 2310 2311 if (sec->sh.sh_entsize != 8) { 2312 static bool warned = false; 2313 if (!warned && opts.verbose) { 2314 WARN("%s: dodgy linker, sh_entsize != 8", sec->name); 2315 warned = true; 2316 } 2317 sec->sh.sh_entsize = 8; 2318 } 2319 2320 if (sec_num_entries(sec) != sec_num_entries(sec->rsec)) { 2321 ERROR("bad .discard.annotate_insn section: missing relocs"); 2322 return -1; 2323 } 2324 2325 for_each_reloc(sec->rsec, reloc) { 2326 type = annotype(file->elf, sec, reloc); 2327 offset = reloc->sym->offset + reloc_addend(reloc); 2328 insn = find_insn(file, reloc->sym->sec, offset); 2329 2330 if (!insn) { 2331 ERROR("bad .discard.annotate_insn entry: %d of type %d", reloc_idx(reloc), type); 2332 return -1; 2333 } 2334 2335 if (func(file, type, insn)) 2336 return -1; 2337 } 2338 2339 return 0; 2340 } 2341 2342 static int __annotate_early(struct objtool_file *file, int type, struct instruction *insn) 2343 { 2344 switch (type) { 2345 2346 /* Must be before add_special_section_alts() */ 2347 case ANNOTYPE_IGNORE_ALTS: 2348 insn->ignore_alts = true; 2349 break; 2350 2351 /* 2352 * Must be before read_unwind_hints() since that needs insn->noendbr. 2353 */ 2354 case ANNOTYPE_NOENDBR: 2355 insn->noendbr = 1; 2356 break; 2357 2358 default: 2359 break; 2360 } 2361 2362 return 0; 2363 } 2364 2365 static int __annotate_ifc(struct objtool_file *file, int type, struct instruction *insn) 2366 { 2367 unsigned long dest_off; 2368 2369 if (type != ANNOTYPE_INTRA_FUNCTION_CALL) 2370 return 0; 2371 2372 if (insn->type != INSN_CALL) { 2373 ERROR_INSN(insn, "intra_function_call not a direct call"); 2374 return -1; 2375 } 2376 2377 /* 2378 * Treat intra-function CALLs as JMPs, but with a stack_op. 2379 * See add_call_destinations(), which strips stack_ops from 2380 * normal CALLs. 2381 */ 2382 insn->type = INSN_JUMP_UNCONDITIONAL; 2383 2384 dest_off = arch_jump_destination(insn); 2385 insn->jump_dest = find_insn(file, insn->sec, dest_off); 2386 if (!insn->jump_dest) { 2387 ERROR_INSN(insn, "can't find call dest at %s+0x%lx", 2388 insn->sec->name, dest_off); 2389 return -1; 2390 } 2391 2392 return 0; 2393 } 2394 2395 static int __annotate_late(struct objtool_file *file, int type, struct instruction *insn) 2396 { 2397 struct symbol *sym; 2398 2399 switch (type) { 2400 case ANNOTYPE_NOENDBR: 2401 /* early */ 2402 break; 2403 2404 case ANNOTYPE_RETPOLINE_SAFE: 2405 if (insn->type != INSN_JUMP_DYNAMIC && 2406 insn->type != INSN_CALL_DYNAMIC && 2407 insn->type != INSN_RETURN && 2408 insn->type != INSN_NOP) { 2409 ERROR_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); 2410 return -1; 2411 } 2412 2413 insn->retpoline_safe = true; 2414 break; 2415 2416 case ANNOTYPE_INSTR_BEGIN: 2417 insn->instr++; 2418 break; 2419 2420 case ANNOTYPE_INSTR_END: 2421 insn->instr--; 2422 break; 2423 2424 case ANNOTYPE_UNRET_BEGIN: 2425 insn->unret = 1; 2426 break; 2427 2428 case ANNOTYPE_IGNORE_ALTS: 2429 /* early */ 2430 break; 2431 2432 case ANNOTYPE_INTRA_FUNCTION_CALL: 2433 /* ifc */ 2434 break; 2435 2436 case ANNOTYPE_REACHABLE: 2437 insn->dead_end = false; 2438 break; 2439 2440 case ANNOTYPE_NOCFI: 2441 sym = insn->sym; 2442 if (!sym) { 2443 ERROR_INSN(insn, "dodgy NOCFI annotation"); 2444 return -1; 2445 } 2446 insn->sym->nocfi = 1; 2447 break; 2448 2449 default: 2450 ERROR_INSN(insn, "Unknown annotation type: %d", type); 2451 return -1; 2452 } 2453 2454 return 0; 2455 } 2456 2457 /* 2458 * Return true if name matches an instrumentation function, where calls to that 2459 * function from noinstr code can safely be removed, but compilers won't do so. 2460 */ 2461 static bool is_profiling_func(const char *name) 2462 { 2463 /* 2464 * Many compilers cannot disable KCOV with a function attribute. 2465 */ 2466 if (!strncmp(name, "__sanitizer_cov_", 16)) 2467 return true; 2468 2469 return false; 2470 } 2471 2472 static int classify_symbols(struct objtool_file *file) 2473 { 2474 struct symbol *func; 2475 size_t len; 2476 2477 for_each_sym(file->elf, func) { 2478 if (is_notype_sym(func) && strstarts(func->name, ".L")) 2479 func->local_label = true; 2480 2481 if (!is_global_sym(func)) 2482 continue; 2483 2484 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, 2485 strlen(STATIC_CALL_TRAMP_PREFIX_STR))) 2486 func->static_call_tramp = true; 2487 2488 if (arch_is_retpoline(func)) 2489 func->retpoline_thunk = true; 2490 2491 if (arch_is_rethunk(func)) 2492 func->return_thunk = true; 2493 2494 if (arch_is_embedded_insn(func)) 2495 func->embedded_insn = true; 2496 2497 if (arch_ftrace_match(func->name)) 2498 func->fentry = true; 2499 2500 if (is_profiling_func(func->name)) 2501 func->profiling_func = true; 2502 2503 len = strlen(func->name); 2504 if (len > sym_name_max_len) 2505 sym_name_max_len = len; 2506 } 2507 2508 return 0; 2509 } 2510 2511 static void mark_rodata(struct objtool_file *file) 2512 { 2513 struct section *sec; 2514 bool found = false; 2515 2516 /* 2517 * Search for the following rodata sections, each of which can 2518 * potentially contain jump tables: 2519 * 2520 * - .rodata: can contain GCC switch tables 2521 * - .rodata.<func>: same, if -fdata-sections is being used 2522 * - .data.rel.ro.c_jump_table: contains C annotated jump tables 2523 * 2524 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2525 */ 2526 for_each_sec(file->elf, sec) { 2527 if ((!strncmp(sec->name, ".rodata", 7) && 2528 !strstr(sec->name, ".str1.")) || 2529 !strncmp(sec->name, ".data.rel.ro", 12)) { 2530 sec->rodata = true; 2531 found = true; 2532 } 2533 } 2534 2535 file->rodata = found; 2536 } 2537 2538 static void mark_holes(struct objtool_file *file) 2539 { 2540 struct instruction *insn; 2541 bool in_hole = false; 2542 2543 if (!opts.link) 2544 return; 2545 2546 /* 2547 * Whole archive runs might encounter dead code from weak symbols. 2548 * This is where the linker will have dropped the weak symbol in 2549 * favour of a regular symbol, but leaves the code in place. 2550 */ 2551 for_each_insn(file, insn) { 2552 if (insn->sym || !find_symbol_hole_containing(insn->sec, insn->offset)) { 2553 in_hole = false; 2554 continue; 2555 } 2556 2557 /* Skip function padding and pfx code */ 2558 if (!in_hole && insn->type == INSN_NOP) 2559 continue; 2560 2561 in_hole = true; 2562 insn->hole = 1; 2563 2564 /* 2565 * If this hole jumps to a .cold function, mark it ignore. 2566 */ 2567 if (insn->jump_dest) { 2568 struct symbol *dest_func = insn_func(insn->jump_dest); 2569 2570 if (dest_func && dest_func->cold) 2571 dest_func->ignore = true; 2572 } 2573 } 2574 } 2575 2576 static bool validate_branch_enabled(void) 2577 { 2578 return opts.stackval || 2579 opts.orc || 2580 opts.uaccess || 2581 opts.checksum; 2582 } 2583 2584 static int decode_sections(struct objtool_file *file) 2585 { 2586 file->klp = is_livepatch_module(file); 2587 2588 mark_rodata(file); 2589 2590 if (init_pv_ops(file)) 2591 return -1; 2592 2593 /* 2594 * Must be before add_{jump_call}_destination. 2595 */ 2596 if (classify_symbols(file)) 2597 return -1; 2598 2599 if (decode_instructions(file)) 2600 return -1; 2601 2602 if (add_ignores(file)) 2603 return -1; 2604 2605 add_uaccess_safe(file); 2606 2607 if (read_annotate(file, __annotate_early)) 2608 return -1; 2609 2610 /* 2611 * Must be before add_jump_destinations(), which depends on 'func' 2612 * being set for alternatives, to enable proper sibling call detection. 2613 */ 2614 if (validate_branch_enabled() || opts.noinstr || opts.hack_jump_label) { 2615 if (add_special_section_alts(file)) 2616 return -1; 2617 } 2618 2619 if (add_jump_destinations(file)) 2620 return -1; 2621 2622 /* 2623 * Must be before add_call_destination(); it changes INSN_CALL to 2624 * INSN_JUMP. 2625 */ 2626 if (read_annotate(file, __annotate_ifc)) 2627 return -1; 2628 2629 if (add_call_destinations(file)) 2630 return -1; 2631 2632 if (add_jump_table_alts(file)) 2633 return -1; 2634 2635 if (read_unwind_hints(file)) 2636 return -1; 2637 2638 /* Must be after add_jump_destinations() */ 2639 mark_holes(file); 2640 2641 /* 2642 * Must be after add_call_destinations() such that it can override 2643 * dead_end_function() marks. 2644 */ 2645 if (read_annotate(file, __annotate_late)) 2646 return -1; 2647 2648 return 0; 2649 } 2650 2651 static bool is_special_call(struct instruction *insn) 2652 { 2653 if (insn->type == INSN_CALL) { 2654 struct symbol *dest = insn_call_dest(insn); 2655 2656 if (!dest) 2657 return false; 2658 2659 if (dest->fentry || dest->embedded_insn) 2660 return true; 2661 } 2662 2663 return false; 2664 } 2665 2666 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state) 2667 { 2668 struct cfi_state *cfi = &state->cfi; 2669 int i; 2670 2671 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap) 2672 return true; 2673 2674 if (cfi->cfa.offset != initial_func_cfi.cfa.offset) 2675 return true; 2676 2677 if (cfi->stack_size != initial_func_cfi.cfa.offset) 2678 return true; 2679 2680 for (i = 0; i < CFI_NUM_REGS; i++) { 2681 if (cfi->regs[i].base != initial_func_cfi.regs[i].base || 2682 cfi->regs[i].offset != initial_func_cfi.regs[i].offset) 2683 return true; 2684 } 2685 2686 return false; 2687 } 2688 2689 static bool check_reg_frame_pos(const struct cfi_reg *reg, 2690 int expected_offset) 2691 { 2692 return reg->base == CFI_CFA && 2693 reg->offset == expected_offset; 2694 } 2695 2696 static bool has_valid_stack_frame(struct insn_state *state) 2697 { 2698 struct cfi_state *cfi = &state->cfi; 2699 2700 if (cfi->cfa.base == CFI_BP && 2701 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) && 2702 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8)) 2703 return true; 2704 2705 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP) 2706 return true; 2707 2708 return false; 2709 } 2710 2711 static int update_cfi_state_regs(struct instruction *insn, 2712 struct cfi_state *cfi, 2713 struct stack_op *op) 2714 { 2715 struct cfi_reg *cfa = &cfi->cfa; 2716 2717 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT) 2718 return 0; 2719 2720 /* push */ 2721 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF) 2722 cfa->offset += 8; 2723 2724 /* pop */ 2725 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF) 2726 cfa->offset -= 8; 2727 2728 /* add immediate to sp */ 2729 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD && 2730 op->dest.reg == CFI_SP && op->src.reg == CFI_SP) 2731 cfa->offset -= op->src.offset; 2732 2733 return 0; 2734 } 2735 2736 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset) 2737 { 2738 if (arch_callee_saved_reg(reg) && 2739 cfi->regs[reg].base == CFI_UNDEFINED) { 2740 cfi->regs[reg].base = base; 2741 cfi->regs[reg].offset = offset; 2742 } 2743 } 2744 2745 static void restore_reg(struct cfi_state *cfi, unsigned char reg) 2746 { 2747 cfi->regs[reg].base = initial_func_cfi.regs[reg].base; 2748 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset; 2749 } 2750 2751 /* 2752 * A note about DRAP stack alignment: 2753 * 2754 * GCC has the concept of a DRAP register, which is used to help keep track of 2755 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP 2756 * register. The typical DRAP pattern is: 2757 * 2758 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10 2759 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp 2760 * 41 ff 72 f8 pushq -0x8(%r10) 2761 * 55 push %rbp 2762 * 48 89 e5 mov %rsp,%rbp 2763 * (more pushes) 2764 * 41 52 push %r10 2765 * ... 2766 * 41 5a pop %r10 2767 * (more pops) 2768 * 5d pop %rbp 2769 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2770 * c3 retq 2771 * 2772 * There are some variations in the epilogues, like: 2773 * 2774 * 5b pop %rbx 2775 * 41 5a pop %r10 2776 * 41 5c pop %r12 2777 * 41 5d pop %r13 2778 * 41 5e pop %r14 2779 * c9 leaveq 2780 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2781 * c3 retq 2782 * 2783 * and: 2784 * 2785 * 4c 8b 55 e8 mov -0x18(%rbp),%r10 2786 * 48 8b 5d e0 mov -0x20(%rbp),%rbx 2787 * 4c 8b 65 f0 mov -0x10(%rbp),%r12 2788 * 4c 8b 6d f8 mov -0x8(%rbp),%r13 2789 * c9 leaveq 2790 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2791 * c3 retq 2792 * 2793 * Sometimes r13 is used as the DRAP register, in which case it's saved and 2794 * restored beforehand: 2795 * 2796 * 41 55 push %r13 2797 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13 2798 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp 2799 * ... 2800 * 49 8d 65 f0 lea -0x10(%r13),%rsp 2801 * 41 5d pop %r13 2802 * c3 retq 2803 */ 2804 static int update_cfi_state(struct instruction *insn, 2805 struct instruction *next_insn, 2806 struct cfi_state *cfi, struct stack_op *op) 2807 { 2808 struct cfi_reg *cfa = &cfi->cfa; 2809 struct cfi_reg *regs = cfi->regs; 2810 2811 /* ignore UNWIND_HINT_UNDEFINED regions */ 2812 if (cfi->force_undefined) 2813 return 0; 2814 2815 /* stack operations don't make sense with an undefined CFA */ 2816 if (cfa->base == CFI_UNDEFINED) { 2817 if (insn_func(insn)) { 2818 WARN_INSN(insn, "undefined stack state"); 2819 return 1; 2820 } 2821 return 0; 2822 } 2823 2824 if (cfi->type == UNWIND_HINT_TYPE_REGS || 2825 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) 2826 return update_cfi_state_regs(insn, cfi, op); 2827 2828 switch (op->dest.type) { 2829 2830 case OP_DEST_REG: 2831 switch (op->src.type) { 2832 2833 case OP_SRC_REG: 2834 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP && 2835 cfa->base == CFI_SP && 2836 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) { 2837 2838 /* mov %rsp, %rbp */ 2839 cfa->base = op->dest.reg; 2840 cfi->bp_scratch = false; 2841 } 2842 2843 else if (op->src.reg == CFI_SP && 2844 op->dest.reg == CFI_BP && cfi->drap) { 2845 2846 /* drap: mov %rsp, %rbp */ 2847 regs[CFI_BP].base = CFI_BP; 2848 regs[CFI_BP].offset = -cfi->stack_size; 2849 cfi->bp_scratch = false; 2850 } 2851 2852 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2853 2854 /* 2855 * mov %rsp, %reg 2856 * 2857 * This is needed for the rare case where GCC 2858 * does: 2859 * 2860 * mov %rsp, %rax 2861 * ... 2862 * mov %rax, %rsp 2863 */ 2864 cfi->vals[op->dest.reg].base = CFI_CFA; 2865 cfi->vals[op->dest.reg].offset = -cfi->stack_size; 2866 } 2867 2868 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP && 2869 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) { 2870 2871 /* 2872 * mov %rbp, %rsp 2873 * 2874 * Restore the original stack pointer (Clang). 2875 */ 2876 cfi->stack_size = -cfi->regs[CFI_BP].offset; 2877 } 2878 2879 else if (op->dest.reg == cfa->base) { 2880 2881 /* mov %reg, %rsp */ 2882 if (cfa->base == CFI_SP && 2883 cfi->vals[op->src.reg].base == CFI_CFA) { 2884 2885 /* 2886 * This is needed for the rare case 2887 * where GCC does something dumb like: 2888 * 2889 * lea 0x8(%rsp), %rcx 2890 * ... 2891 * mov %rcx, %rsp 2892 */ 2893 cfa->offset = -cfi->vals[op->src.reg].offset; 2894 cfi->stack_size = cfa->offset; 2895 2896 } else if (cfa->base == CFI_SP && 2897 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2898 cfi->vals[op->src.reg].offset == cfa->offset) { 2899 2900 /* 2901 * Stack swizzle: 2902 * 2903 * 1: mov %rsp, (%[tos]) 2904 * 2: mov %[tos], %rsp 2905 * ... 2906 * 3: pop %rsp 2907 * 2908 * Where: 2909 * 2910 * 1 - places a pointer to the previous 2911 * stack at the Top-of-Stack of the 2912 * new stack. 2913 * 2914 * 2 - switches to the new stack. 2915 * 2916 * 3 - pops the Top-of-Stack to restore 2917 * the original stack. 2918 * 2919 * Note: we set base to SP_INDIRECT 2920 * here and preserve offset. Therefore 2921 * when the unwinder reaches ToS it 2922 * will dereference SP and then add the 2923 * offset to find the next frame, IOW: 2924 * (%rsp) + offset. 2925 */ 2926 cfa->base = CFI_SP_INDIRECT; 2927 2928 } else { 2929 cfa->base = CFI_UNDEFINED; 2930 cfa->offset = 0; 2931 } 2932 } 2933 2934 else if (op->dest.reg == CFI_SP && 2935 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2936 cfi->vals[op->src.reg].offset == cfa->offset) { 2937 2938 /* 2939 * The same stack swizzle case 2) as above. But 2940 * because we can't change cfa->base, case 3) 2941 * will become a regular POP. Pretend we're a 2942 * PUSH so things don't go unbalanced. 2943 */ 2944 cfi->stack_size += 8; 2945 } 2946 2947 2948 break; 2949 2950 case OP_SRC_ADD: 2951 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) { 2952 2953 /* add imm, %rsp */ 2954 cfi->stack_size -= op->src.offset; 2955 if (cfa->base == CFI_SP) 2956 cfa->offset -= op->src.offset; 2957 break; 2958 } 2959 2960 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP && 2961 insn->sym->frame_pointer) { 2962 /* addi.d fp,sp,imm on LoongArch */ 2963 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) { 2964 cfa->base = CFI_BP; 2965 cfa->offset = 0; 2966 } 2967 break; 2968 } 2969 2970 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { 2971 /* addi.d sp,fp,imm on LoongArch */ 2972 if (cfa->base == CFI_BP && cfa->offset == 0) { 2973 if (insn->sym->frame_pointer) { 2974 cfa->base = CFI_SP; 2975 cfa->offset = -op->src.offset; 2976 } 2977 } else { 2978 /* lea disp(%rbp), %rsp */ 2979 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); 2980 } 2981 break; 2982 } 2983 2984 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2985 2986 /* drap: lea disp(%rsp), %drap */ 2987 cfi->drap_reg = op->dest.reg; 2988 2989 /* 2990 * lea disp(%rsp), %reg 2991 * 2992 * This is needed for the rare case where GCC 2993 * does something dumb like: 2994 * 2995 * lea 0x8(%rsp), %rcx 2996 * ... 2997 * mov %rcx, %rsp 2998 */ 2999 cfi->vals[op->dest.reg].base = CFI_CFA; 3000 cfi->vals[op->dest.reg].offset = \ 3001 -cfi->stack_size + op->src.offset; 3002 3003 break; 3004 } 3005 3006 if (cfi->drap && op->dest.reg == CFI_SP && 3007 op->src.reg == cfi->drap_reg) { 3008 3009 /* drap: lea disp(%drap), %rsp */ 3010 cfa->base = CFI_SP; 3011 cfa->offset = cfi->stack_size = -op->src.offset; 3012 cfi->drap_reg = CFI_UNDEFINED; 3013 cfi->drap = false; 3014 break; 3015 } 3016 3017 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { 3018 WARN_INSN(insn, "unsupported stack register modification"); 3019 return -1; 3020 } 3021 3022 break; 3023 3024 case OP_SRC_AND: 3025 if (op->dest.reg != CFI_SP || 3026 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || 3027 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { 3028 WARN_INSN(insn, "unsupported stack pointer realignment"); 3029 return -1; 3030 } 3031 3032 if (cfi->drap_reg != CFI_UNDEFINED) { 3033 /* drap: and imm, %rsp */ 3034 cfa->base = cfi->drap_reg; 3035 cfa->offset = cfi->stack_size = 0; 3036 cfi->drap = true; 3037 } 3038 3039 /* 3040 * Older versions of GCC (4.8ish) realign the stack 3041 * without DRAP, with a frame pointer. 3042 */ 3043 3044 break; 3045 3046 case OP_SRC_POP: 3047 case OP_SRC_POPF: 3048 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) { 3049 3050 /* pop %rsp; # restore from a stack swizzle */ 3051 cfa->base = CFI_SP; 3052 break; 3053 } 3054 3055 if (!cfi->drap && op->dest.reg == cfa->base) { 3056 3057 /* pop %rbp */ 3058 cfa->base = CFI_SP; 3059 } 3060 3061 if (cfi->drap && cfa->base == CFI_BP_INDIRECT && 3062 op->dest.reg == cfi->drap_reg && 3063 cfi->drap_offset == -cfi->stack_size) { 3064 3065 /* drap: pop %drap */ 3066 cfa->base = cfi->drap_reg; 3067 cfa->offset = 0; 3068 cfi->drap_offset = -1; 3069 3070 } else if (cfi->stack_size == -regs[op->dest.reg].offset) { 3071 3072 /* pop %reg */ 3073 restore_reg(cfi, op->dest.reg); 3074 } 3075 3076 cfi->stack_size -= 8; 3077 if (cfa->base == CFI_SP) 3078 cfa->offset -= 8; 3079 3080 break; 3081 3082 case OP_SRC_REG_INDIRECT: 3083 if (!cfi->drap && op->dest.reg == cfa->base && 3084 op->dest.reg == CFI_BP) { 3085 3086 /* mov disp(%rsp), %rbp */ 3087 cfa->base = CFI_SP; 3088 cfa->offset = cfi->stack_size; 3089 } 3090 3091 if (cfi->drap && op->src.reg == CFI_BP && 3092 op->src.offset == cfi->drap_offset) { 3093 3094 /* drap: mov disp(%rbp), %drap */ 3095 cfa->base = cfi->drap_reg; 3096 cfa->offset = 0; 3097 cfi->drap_offset = -1; 3098 } 3099 3100 if (cfi->drap && op->src.reg == CFI_BP && 3101 op->src.offset == regs[op->dest.reg].offset) { 3102 3103 /* drap: mov disp(%rbp), %reg */ 3104 restore_reg(cfi, op->dest.reg); 3105 3106 } else if (op->src.reg == cfa->base && 3107 op->src.offset == regs[op->dest.reg].offset + cfa->offset) { 3108 3109 /* mov disp(%rbp), %reg */ 3110 /* mov disp(%rsp), %reg */ 3111 restore_reg(cfi, op->dest.reg); 3112 3113 } else if (op->src.reg == CFI_SP && 3114 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) { 3115 3116 /* mov disp(%rsp), %reg */ 3117 restore_reg(cfi, op->dest.reg); 3118 } 3119 3120 break; 3121 3122 default: 3123 WARN_INSN(insn, "unknown stack-related instruction"); 3124 return -1; 3125 } 3126 3127 break; 3128 3129 case OP_DEST_PUSH: 3130 case OP_DEST_PUSHF: 3131 cfi->stack_size += 8; 3132 if (cfa->base == CFI_SP) 3133 cfa->offset += 8; 3134 3135 if (op->src.type != OP_SRC_REG) 3136 break; 3137 3138 if (cfi->drap) { 3139 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3140 3141 /* drap: push %drap */ 3142 cfa->base = CFI_BP_INDIRECT; 3143 cfa->offset = -cfi->stack_size; 3144 3145 /* save drap so we know when to restore it */ 3146 cfi->drap_offset = -cfi->stack_size; 3147 3148 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) { 3149 3150 /* drap: push %rbp */ 3151 cfi->stack_size = 0; 3152 3153 } else { 3154 3155 /* drap: push %reg */ 3156 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); 3157 } 3158 3159 } else { 3160 3161 /* push %reg */ 3162 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size); 3163 } 3164 3165 /* detect when asm code uses rbp as a scratch register */ 3166 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP && 3167 cfa->base != CFI_BP) 3168 cfi->bp_scratch = true; 3169 break; 3170 3171 case OP_DEST_REG_INDIRECT: 3172 3173 if (cfi->drap) { 3174 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3175 3176 /* drap: mov %drap, disp(%rbp) */ 3177 cfa->base = CFI_BP_INDIRECT; 3178 cfa->offset = op->dest.offset; 3179 3180 /* save drap offset so we know when to restore it */ 3181 cfi->drap_offset = op->dest.offset; 3182 } else { 3183 3184 /* drap: mov reg, disp(%rbp) */ 3185 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); 3186 } 3187 3188 } else if (op->dest.reg == cfa->base) { 3189 3190 /* mov reg, disp(%rbp) */ 3191 /* mov reg, disp(%rsp) */ 3192 save_reg(cfi, op->src.reg, CFI_CFA, 3193 op->dest.offset - cfi->cfa.offset); 3194 3195 } else if (op->dest.reg == CFI_SP) { 3196 3197 /* mov reg, disp(%rsp) */ 3198 save_reg(cfi, op->src.reg, CFI_CFA, 3199 op->dest.offset - cfi->stack_size); 3200 3201 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) { 3202 3203 /* mov %rsp, (%reg); # setup a stack swizzle. */ 3204 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT; 3205 cfi->vals[op->dest.reg].offset = cfa->offset; 3206 } 3207 3208 break; 3209 3210 case OP_DEST_MEM: 3211 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { 3212 WARN_INSN(insn, "unknown stack-related memory operation"); 3213 return -1; 3214 } 3215 3216 /* pop mem */ 3217 cfi->stack_size -= 8; 3218 if (cfa->base == CFI_SP) 3219 cfa->offset -= 8; 3220 3221 break; 3222 3223 default: 3224 WARN_INSN(insn, "unknown stack-related instruction"); 3225 return -1; 3226 } 3227 3228 return 0; 3229 } 3230 3231 /* 3232 * The stack layouts of alternatives instructions can sometimes diverge when 3233 * they have stack modifications. That's fine as long as the potential stack 3234 * layouts don't conflict at any given potential instruction boundary. 3235 * 3236 * Flatten the CFIs of the different alternative code streams (both original 3237 * and replacement) into a single shared CFI array which can be used to detect 3238 * conflicts and nicely feed a linear array of ORC entries to the unwinder. 3239 */ 3240 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn) 3241 { 3242 struct cfi_state **alt_cfi; 3243 int group_off; 3244 3245 if (!insn->alt_group) 3246 return 0; 3247 3248 if (!insn->cfi) { 3249 WARN("CFI missing"); 3250 return -1; 3251 } 3252 3253 alt_cfi = insn->alt_group->cfi; 3254 group_off = insn->offset - insn->alt_group->first_insn->offset; 3255 3256 if (!alt_cfi[group_off]) { 3257 alt_cfi[group_off] = insn->cfi; 3258 } else { 3259 if (cficmp(alt_cfi[group_off], insn->cfi)) { 3260 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group; 3261 struct instruction *orig = orig_group->first_insn; 3262 WARN_INSN(orig, "stack layout conflict in alternatives: %s", 3263 offstr(insn->sec, insn->offset)); 3264 return -1; 3265 } 3266 } 3267 3268 return 0; 3269 } 3270 3271 static int handle_insn_ops(struct instruction *insn, 3272 struct instruction *next_insn, 3273 struct insn_state *state) 3274 { 3275 struct stack_op *op; 3276 int ret; 3277 3278 for (op = insn->stack_ops; op; op = op->next) { 3279 3280 ret = update_cfi_state(insn, next_insn, &state->cfi, op); 3281 if (ret) 3282 return ret; 3283 3284 if (!opts.uaccess || !insn->alt_group) 3285 continue; 3286 3287 if (op->dest.type == OP_DEST_PUSHF) { 3288 if (!state->uaccess_stack) { 3289 state->uaccess_stack = 1; 3290 } else if (state->uaccess_stack >> 31) { 3291 WARN_INSN(insn, "PUSHF stack exhausted"); 3292 return 1; 3293 } 3294 state->uaccess_stack <<= 1; 3295 state->uaccess_stack |= state->uaccess; 3296 } 3297 3298 if (op->src.type == OP_SRC_POPF) { 3299 if (state->uaccess_stack) { 3300 state->uaccess = state->uaccess_stack & 1; 3301 state->uaccess_stack >>= 1; 3302 if (state->uaccess_stack == 1) 3303 state->uaccess_stack = 0; 3304 } 3305 } 3306 } 3307 3308 return 0; 3309 } 3310 3311 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) 3312 { 3313 struct cfi_state *cfi1 = insn->cfi; 3314 int i; 3315 3316 if (!cfi1) { 3317 WARN("CFI missing"); 3318 return false; 3319 } 3320 3321 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { 3322 3323 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d", 3324 cfi1->cfa.base, cfi1->cfa.offset, 3325 cfi2->cfa.base, cfi2->cfa.offset); 3326 return false; 3327 3328 } 3329 3330 if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) { 3331 for (i = 0; i < CFI_NUM_REGS; i++) { 3332 3333 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], sizeof(struct cfi_reg))) 3334 continue; 3335 3336 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", 3337 i, cfi1->regs[i].base, cfi1->regs[i].offset, 3338 i, cfi2->regs[i].base, cfi2->regs[i].offset); 3339 } 3340 return false; 3341 } 3342 3343 if (cfi1->type != cfi2->type) { 3344 3345 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d", 3346 cfi1->type, cfi2->type); 3347 return false; 3348 } 3349 3350 if (cfi1->drap != cfi2->drap || 3351 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || 3352 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { 3353 3354 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", 3355 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, 3356 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); 3357 return false; 3358 } 3359 3360 return true; 3361 } 3362 3363 static inline bool func_uaccess_safe(struct symbol *func) 3364 { 3365 if (func) 3366 return func->uaccess_safe; 3367 3368 return false; 3369 } 3370 3371 static inline const char *call_dest_name(struct instruction *insn) 3372 { 3373 static char pvname[19]; 3374 struct reloc *reloc; 3375 int idx; 3376 3377 if (insn_call_dest(insn)) 3378 return insn_call_dest(insn)->name; 3379 3380 reloc = insn_reloc(NULL, insn); 3381 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) { 3382 idx = (reloc_addend(reloc) / sizeof(void *)); 3383 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx); 3384 return pvname; 3385 } 3386 3387 return "{dynamic}"; 3388 } 3389 3390 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn) 3391 { 3392 struct symbol *target; 3393 struct reloc *reloc; 3394 int idx; 3395 3396 reloc = insn_reloc(file, insn); 3397 if (!reloc || strcmp(reloc->sym->name, "pv_ops")) 3398 return false; 3399 3400 idx = arch_insn_adjusted_addend(insn, reloc) / sizeof(void *); 3401 3402 if (file->pv_ops[idx].clean) 3403 return true; 3404 3405 file->pv_ops[idx].clean = true; 3406 3407 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) { 3408 if (!target->sec->noinstr) { 3409 WARN("pv_ops[%d]: %s", idx, target->name); 3410 file->pv_ops[idx].clean = false; 3411 } 3412 } 3413 3414 return file->pv_ops[idx].clean; 3415 } 3416 3417 static inline bool noinstr_call_dest(struct objtool_file *file, 3418 struct instruction *insn, 3419 struct symbol *func) 3420 { 3421 /* 3422 * We can't deal with indirect function calls at present; 3423 * assume they're instrumented. 3424 */ 3425 if (!func) { 3426 if (file->pv_ops) 3427 return pv_call_dest(file, insn); 3428 3429 return false; 3430 } 3431 3432 /* 3433 * If the symbol is from a noinstr section; we good. 3434 */ 3435 if (func->sec->noinstr) 3436 return true; 3437 3438 /* 3439 * If the symbol is a static_call trampoline, we can't tell. 3440 */ 3441 if (func->static_call_tramp) 3442 return true; 3443 3444 /* 3445 * The __ubsan_handle_*() calls are like WARN(), they only happen when 3446 * something 'BAD' happened. At the risk of taking the machine down, 3447 * let them proceed to get the message out. 3448 */ 3449 if (!strncmp(func->name, "__ubsan_handle_", 15)) 3450 return true; 3451 3452 return false; 3453 } 3454 3455 static int validate_call(struct objtool_file *file, 3456 struct instruction *insn, 3457 struct insn_state *state) 3458 { 3459 if (state->noinstr && state->instr <= 0 && 3460 !noinstr_call_dest(file, insn, insn_call_dest(insn))) { 3461 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn)); 3462 return 1; 3463 } 3464 3465 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) { 3466 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn)); 3467 return 1; 3468 } 3469 3470 if (state->df) { 3471 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn)); 3472 return 1; 3473 } 3474 3475 return 0; 3476 } 3477 3478 static int validate_sibling_call(struct objtool_file *file, 3479 struct instruction *insn, 3480 struct insn_state *state) 3481 { 3482 if (insn_func(insn) && has_modified_stack_frame(insn, state)) { 3483 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame"); 3484 return 1; 3485 } 3486 3487 return validate_call(file, insn, state); 3488 } 3489 3490 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 3491 { 3492 if (state->noinstr && state->instr > 0) { 3493 WARN_INSN(insn, "return with instrumentation enabled"); 3494 return 1; 3495 } 3496 3497 if (state->uaccess && !func_uaccess_safe(func)) { 3498 WARN_INSN(insn, "return with UACCESS enabled"); 3499 return 1; 3500 } 3501 3502 if (!state->uaccess && func_uaccess_safe(func)) { 3503 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function"); 3504 return 1; 3505 } 3506 3507 if (state->df) { 3508 WARN_INSN(insn, "return with DF set"); 3509 return 1; 3510 } 3511 3512 if (func && has_modified_stack_frame(insn, state)) { 3513 WARN_INSN(insn, "return with modified stack frame"); 3514 return 1; 3515 } 3516 3517 if (state->cfi.bp_scratch) { 3518 WARN_INSN(insn, "BP used as a scratch register"); 3519 return 1; 3520 } 3521 3522 return 0; 3523 } 3524 3525 static struct instruction *next_insn_to_validate(struct objtool_file *file, 3526 struct instruction *insn) 3527 { 3528 struct alt_group *alt_group = insn->alt_group; 3529 3530 /* 3531 * Simulate the fact that alternatives are patched in-place. When the 3532 * end of a replacement alt_group is reached, redirect objtool flow to 3533 * the end of the original alt_group. 3534 * 3535 * insn->alts->insn -> alt_group->first_insn 3536 * ... 3537 * alt_group->last_insn 3538 * [alt_group->nop] -> next(orig_group->last_insn) 3539 */ 3540 if (alt_group) { 3541 if (alt_group->nop) { 3542 /* ->nop implies ->orig_group */ 3543 if (insn == alt_group->last_insn) 3544 return alt_group->nop; 3545 if (insn == alt_group->nop) 3546 goto next_orig; 3547 } 3548 if (insn == alt_group->last_insn && alt_group->orig_group) 3549 goto next_orig; 3550 } 3551 3552 return next_insn_same_sec(file, insn); 3553 3554 next_orig: 3555 return next_insn_same_sec(file, alt_group->orig_group->last_insn); 3556 } 3557 3558 static bool skip_alt_group(struct instruction *insn) 3559 { 3560 struct instruction *alt_insn = insn->alts ? insn->alts->insn : NULL; 3561 3562 if (!insn->alt_group) 3563 return false; 3564 3565 /* ANNOTATE_IGNORE_ALTERNATIVE */ 3566 if (insn->alt_group->ignore) { 3567 TRACE_INSN(insn, "alt group ignored"); 3568 return true; 3569 } 3570 3571 /* 3572 * For NOP patched with CLAC/STAC, only follow the latter to avoid 3573 * impossible code paths combining patched CLAC with unpatched STAC 3574 * or vice versa. 3575 * 3576 * ANNOTATE_IGNORE_ALTERNATIVE could have been used here, but Linus 3577 * requested not to do that to avoid hurting .s file readability 3578 * around CLAC/STAC alternative sites. 3579 */ 3580 3581 if (!alt_insn) 3582 return false; 3583 3584 /* Don't override ASM_{CLAC,STAC}_UNSAFE */ 3585 if (alt_insn->alt_group && alt_insn->alt_group->ignore) 3586 return false; 3587 3588 return alt_insn->type == INSN_CLAC || alt_insn->type == INSN_STAC; 3589 } 3590 3591 static int checksum_debug_init(struct objtool_file *file) 3592 { 3593 char *dup, *s; 3594 3595 if (!opts.debug_checksum) 3596 return 0; 3597 3598 dup = strdup(opts.debug_checksum); 3599 if (!dup) { 3600 ERROR_GLIBC("strdup"); 3601 return -1; 3602 } 3603 3604 s = dup; 3605 while (*s) { 3606 struct symbol *func; 3607 char *comma; 3608 3609 comma = strchr(s, ','); 3610 if (comma) 3611 *comma = '\0'; 3612 3613 func = find_symbol_by_name(file->elf, s); 3614 if (!func || !is_func_sym(func)) 3615 WARN("--debug-checksum: can't find '%s'", s); 3616 else 3617 func->debug_checksum = 1; 3618 3619 if (!comma) 3620 break; 3621 3622 s = comma + 1; 3623 } 3624 3625 free(dup); 3626 return 0; 3627 } 3628 3629 static void checksum_update_insn(struct objtool_file *file, struct symbol *func, 3630 struct instruction *insn) 3631 { 3632 struct reloc *reloc = insn_reloc(file, insn); 3633 unsigned long offset; 3634 struct symbol *sym; 3635 3636 if (insn->fake) 3637 return; 3638 3639 checksum_update(func, insn, insn->sec->data->d_buf + insn->offset, insn->len); 3640 3641 if (!reloc) { 3642 struct symbol *call_dest = insn_call_dest(insn); 3643 3644 if (call_dest) 3645 checksum_update(func, insn, call_dest->demangled_name, 3646 strlen(call_dest->demangled_name)); 3647 return; 3648 } 3649 3650 sym = reloc->sym; 3651 offset = arch_insn_adjusted_addend(insn, reloc); 3652 3653 if (is_string_sec(sym->sec)) { 3654 char *str; 3655 3656 str = sym->sec->data->d_buf + sym->offset + offset; 3657 checksum_update(func, insn, str, strlen(str)); 3658 return; 3659 } 3660 3661 if (is_sec_sym(sym)) { 3662 sym = find_symbol_containing(reloc->sym->sec, offset); 3663 if (!sym) 3664 return; 3665 3666 offset -= sym->offset; 3667 } 3668 3669 checksum_update(func, insn, sym->demangled_name, strlen(sym->demangled_name)); 3670 checksum_update(func, insn, &offset, sizeof(offset)); 3671 } 3672 3673 static int validate_branch(struct objtool_file *file, struct symbol *func, 3674 struct instruction *insn, struct insn_state state); 3675 static int do_validate_branch(struct objtool_file *file, struct symbol *func, 3676 struct instruction *insn, struct insn_state state); 3677 3678 static int validate_insn(struct objtool_file *file, struct symbol *func, 3679 struct instruction *insn, struct insn_state *statep, 3680 struct instruction *prev_insn, struct instruction *next_insn, 3681 bool *dead_end) 3682 { 3683 /* prev_state is not used if there is no disassembly support */ 3684 struct insn_state prev_state __maybe_unused; 3685 struct alternative *alt; 3686 u8 visited; 3687 int ret; 3688 3689 /* 3690 * Any returns before the end of this function are effectively dead 3691 * ends, i.e. validate_branch() has reached the end of the branch. 3692 */ 3693 *dead_end = true; 3694 3695 visited = VISITED_BRANCH << statep->uaccess; 3696 if (insn->visited & VISITED_BRANCH_MASK) { 3697 if (!insn->hint && !insn_cfi_match(insn, &statep->cfi)) 3698 return 1; 3699 3700 if (insn->visited & visited) { 3701 TRACE_INSN(insn, "already visited"); 3702 return 0; 3703 } 3704 } else { 3705 nr_insns_visited++; 3706 } 3707 3708 if (statep->noinstr) 3709 statep->instr += insn->instr; 3710 3711 if (insn->hint) { 3712 if (insn->restore) { 3713 struct instruction *save_insn, *i; 3714 3715 i = insn; 3716 save_insn = NULL; 3717 3718 sym_for_each_insn_continue_reverse(file, func, i) { 3719 if (i->save) { 3720 save_insn = i; 3721 break; 3722 } 3723 } 3724 3725 if (!save_insn) { 3726 WARN_INSN(insn, "no corresponding CFI save for CFI restore"); 3727 return 1; 3728 } 3729 3730 if (!save_insn->visited) { 3731 /* 3732 * If the restore hint insn is at the 3733 * beginning of a basic block and was 3734 * branched to from elsewhere, and the 3735 * save insn hasn't been visited yet, 3736 * defer following this branch for now. 3737 * It will be seen later via the 3738 * straight-line path. 3739 */ 3740 if (!prev_insn) { 3741 TRACE_INSN(insn, "defer restore"); 3742 return 0; 3743 } 3744 3745 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo"); 3746 return 1; 3747 } 3748 3749 insn->cfi = save_insn->cfi; 3750 nr_cfi_reused++; 3751 } 3752 3753 statep->cfi = *insn->cfi; 3754 } else { 3755 /* XXX track if we actually changed statep->cfi */ 3756 3757 if (prev_insn && !cficmp(prev_insn->cfi, &statep->cfi)) { 3758 insn->cfi = prev_insn->cfi; 3759 nr_cfi_reused++; 3760 } else { 3761 insn->cfi = cfi_hash_find_or_add(&statep->cfi); 3762 } 3763 } 3764 3765 insn->visited |= visited; 3766 3767 if (propagate_alt_cfi(file, insn)) 3768 return 1; 3769 3770 if (insn->alts) { 3771 int i, num_alts; 3772 3773 num_alts = 0; 3774 for (alt = insn->alts; alt; alt = alt->next) 3775 num_alts++; 3776 3777 i = 1; 3778 for (alt = insn->alts; alt; alt = alt->next) { 3779 TRACE_INSN(insn, "alternative %d/%d", i, num_alts); 3780 ret = validate_branch(file, func, alt->insn, *statep); 3781 if (ret) { 3782 BT_INSN(insn, "(alt)"); 3783 return ret; 3784 } 3785 i++; 3786 } 3787 TRACE_INSN(insn, "alternative DEFAULT"); 3788 } 3789 3790 if (skip_alt_group(insn)) 3791 return 0; 3792 3793 prev_state = *statep; 3794 ret = handle_insn_ops(insn, next_insn, statep); 3795 TRACE_INSN_STATE(insn, &prev_state, statep); 3796 3797 if (ret) 3798 return 1; 3799 3800 switch (insn->type) { 3801 3802 case INSN_RETURN: 3803 TRACE_INSN(insn, "return"); 3804 return validate_return(func, insn, statep); 3805 3806 case INSN_CALL: 3807 case INSN_CALL_DYNAMIC: 3808 if (insn->type == INSN_CALL) 3809 TRACE_INSN(insn, "call"); 3810 else 3811 TRACE_INSN(insn, "indirect call"); 3812 3813 ret = validate_call(file, insn, statep); 3814 if (ret) 3815 return ret; 3816 3817 if (opts.stackval && func && !is_special_call(insn) && 3818 !has_valid_stack_frame(statep)) { 3819 WARN_INSN(insn, "call without frame pointer save/setup"); 3820 return 1; 3821 } 3822 3823 break; 3824 3825 case INSN_JUMP_CONDITIONAL: 3826 case INSN_JUMP_UNCONDITIONAL: 3827 if (is_sibling_call(insn)) { 3828 TRACE_INSN(insn, "sibling call"); 3829 ret = validate_sibling_call(file, insn, statep); 3830 if (ret) 3831 return ret; 3832 3833 } else if (insn->jump_dest) { 3834 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3835 TRACE_INSN(insn, "unconditional jump"); 3836 else 3837 TRACE_INSN(insn, "jump taken"); 3838 3839 ret = validate_branch(file, func, insn->jump_dest, *statep); 3840 if (ret) { 3841 BT_INSN(insn, "(branch)"); 3842 return ret; 3843 } 3844 } 3845 3846 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3847 return 0; 3848 3849 TRACE_INSN(insn, "jump not taken"); 3850 break; 3851 3852 case INSN_JUMP_DYNAMIC: 3853 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3854 TRACE_INSN(insn, "indirect jump"); 3855 if (is_sibling_call(insn)) { 3856 ret = validate_sibling_call(file, insn, statep); 3857 if (ret) 3858 return ret; 3859 } 3860 3861 if (insn->type == INSN_JUMP_DYNAMIC) 3862 return 0; 3863 3864 break; 3865 3866 case INSN_SYSCALL: 3867 TRACE_INSN(insn, "syscall"); 3868 if (func && (!next_insn || !next_insn->hint)) { 3869 WARN_INSN(insn, "unsupported instruction in callable function"); 3870 return 1; 3871 } 3872 3873 break; 3874 3875 case INSN_SYSRET: 3876 TRACE_INSN(insn, "sysret"); 3877 if (func && (!next_insn || !next_insn->hint)) { 3878 WARN_INSN(insn, "unsupported instruction in callable function"); 3879 return 1; 3880 } 3881 3882 return 0; 3883 3884 case INSN_STAC: 3885 TRACE_INSN(insn, "stac"); 3886 if (!opts.uaccess) 3887 break; 3888 3889 if (statep->uaccess) { 3890 WARN_INSN(insn, "recursive UACCESS enable"); 3891 return 1; 3892 } 3893 3894 statep->uaccess = true; 3895 break; 3896 3897 case INSN_CLAC: 3898 TRACE_INSN(insn, "clac"); 3899 if (!opts.uaccess) 3900 break; 3901 3902 if (!statep->uaccess && func) { 3903 WARN_INSN(insn, "redundant UACCESS disable"); 3904 return 1; 3905 } 3906 3907 if (func_uaccess_safe(func) && !statep->uaccess_stack) { 3908 WARN_INSN(insn, "UACCESS-safe disables UACCESS"); 3909 return 1; 3910 } 3911 3912 statep->uaccess = false; 3913 break; 3914 3915 case INSN_STD: 3916 TRACE_INSN(insn, "std"); 3917 if (statep->df) { 3918 WARN_INSN(insn, "recursive STD"); 3919 return 1; 3920 } 3921 3922 statep->df = true; 3923 break; 3924 3925 case INSN_CLD: 3926 TRACE_INSN(insn, "cld"); 3927 if (!statep->df && func) { 3928 WARN_INSN(insn, "redundant CLD"); 3929 return 1; 3930 } 3931 3932 statep->df = false; 3933 break; 3934 3935 default: 3936 break; 3937 } 3938 3939 if (insn->dead_end) 3940 TRACE_INSN(insn, "dead end"); 3941 3942 *dead_end = insn->dead_end; 3943 return 0; 3944 } 3945 3946 /* 3947 * Follow the branch starting at the given instruction, and recursively follow 3948 * any other branches (jumps). Meanwhile, track the frame pointer state at 3949 * each instruction and validate all the rules described in 3950 * tools/objtool/Documentation/objtool.txt. 3951 */ 3952 static int do_validate_branch(struct objtool_file *file, struct symbol *func, 3953 struct instruction *insn, struct insn_state state) 3954 { 3955 struct instruction *next_insn, *prev_insn = NULL; 3956 bool dead_end; 3957 int ret; 3958 3959 if (func && func->ignore) 3960 return 0; 3961 3962 do { 3963 insn->trace = 0; 3964 next_insn = next_insn_to_validate(file, insn); 3965 3966 if (opts.checksum && func && insn->sec) 3967 checksum_update_insn(file, func, insn); 3968 3969 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { 3970 /* Ignore KCFI type preambles, which always fall through */ 3971 if (is_prefix_func(func)) 3972 return 0; 3973 3974 if (file->ignore_unreachables) 3975 return 0; 3976 3977 WARN("%s() falls through to next function %s()", 3978 func->name, insn_func(insn)->name); 3979 func->warned = 1; 3980 3981 return 1; 3982 } 3983 3984 ret = validate_insn(file, func, insn, &state, prev_insn, next_insn, 3985 &dead_end); 3986 3987 if (!insn->trace) { 3988 if (ret) 3989 TRACE_INSN(insn, "warning (%d)", ret); 3990 else 3991 TRACE_INSN(insn, NULL); 3992 } 3993 3994 if (!dead_end && !next_insn) { 3995 if (state.cfi.cfa.base == CFI_UNDEFINED) 3996 return 0; 3997 if (file->ignore_unreachables) 3998 return 0; 3999 4000 WARN("%s%sunexpected end of section %s", 4001 func ? func->name : "", func ? "(): " : "", 4002 insn->sec->name); 4003 return 1; 4004 } 4005 4006 prev_insn = insn; 4007 insn = next_insn; 4008 4009 } while (!dead_end); 4010 4011 return ret; 4012 } 4013 4014 static int validate_branch(struct objtool_file *file, struct symbol *func, 4015 struct instruction *insn, struct insn_state state) 4016 { 4017 int ret; 4018 4019 trace_depth_inc(); 4020 ret = do_validate_branch(file, func, insn, state); 4021 trace_depth_dec(); 4022 4023 return ret; 4024 } 4025 4026 static int validate_unwind_hint(struct objtool_file *file, 4027 struct instruction *insn, 4028 struct insn_state *state) 4029 { 4030 if (insn->hint && !insn->visited) { 4031 struct symbol *func = insn_func(insn); 4032 int ret; 4033 4034 if (opts.checksum) 4035 checksum_init(func); 4036 4037 ret = validate_branch(file, func, insn, *state); 4038 if (ret) 4039 BT_INSN(insn, "<=== (hint)"); 4040 return ret; 4041 } 4042 4043 return 0; 4044 } 4045 4046 static int validate_unwind_hints(struct objtool_file *file, struct section *sec) 4047 { 4048 struct instruction *insn; 4049 struct insn_state state; 4050 int warnings = 0; 4051 4052 if (!file->hints) 4053 return 0; 4054 4055 init_insn_state(file, &state, sec); 4056 4057 if (sec) { 4058 sec_for_each_insn(file, sec, insn) 4059 warnings += validate_unwind_hint(file, insn, &state); 4060 } else { 4061 for_each_insn(file, insn) 4062 warnings += validate_unwind_hint(file, insn, &state); 4063 } 4064 4065 return warnings; 4066 } 4067 4068 /* 4069 * Validate rethunk entry constraint: must untrain RET before the first RET. 4070 * 4071 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes 4072 * before an actual RET instruction. 4073 */ 4074 static int validate_unret(struct objtool_file *file, struct instruction *insn) 4075 { 4076 struct instruction *next, *dest; 4077 int ret; 4078 4079 for (;;) { 4080 next = next_insn_to_validate(file, insn); 4081 4082 if (insn->visited & VISITED_UNRET) 4083 return 0; 4084 4085 insn->visited |= VISITED_UNRET; 4086 4087 if (insn->alts) { 4088 struct alternative *alt; 4089 for (alt = insn->alts; alt; alt = alt->next) { 4090 ret = validate_unret(file, alt->insn); 4091 if (ret) { 4092 BT_INSN(insn, "(alt)"); 4093 return ret; 4094 } 4095 } 4096 } 4097 4098 switch (insn->type) { 4099 4100 case INSN_CALL_DYNAMIC: 4101 case INSN_JUMP_DYNAMIC: 4102 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4103 WARN_INSN(insn, "early indirect call"); 4104 return 1; 4105 4106 case INSN_JUMP_UNCONDITIONAL: 4107 case INSN_JUMP_CONDITIONAL: 4108 if (!is_sibling_call(insn)) { 4109 if (!insn->jump_dest) { 4110 WARN_INSN(insn, "unresolved jump target after linking?!?"); 4111 return 1; 4112 } 4113 ret = validate_unret(file, insn->jump_dest); 4114 if (ret) { 4115 BT_INSN(insn, "(branch%s)", 4116 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : ""); 4117 return ret; 4118 } 4119 4120 if (insn->type == INSN_JUMP_UNCONDITIONAL) 4121 return 0; 4122 4123 break; 4124 } 4125 4126 /* fallthrough */ 4127 case INSN_CALL: 4128 dest = find_insn(file, insn_call_dest(insn)->sec, 4129 insn_call_dest(insn)->offset); 4130 if (!dest) { 4131 WARN("Unresolved function after linking!?: %s", 4132 insn_call_dest(insn)->name); 4133 return 1; 4134 } 4135 4136 ret = validate_unret(file, dest); 4137 if (ret) { 4138 BT_INSN(insn, "(call)"); 4139 return ret; 4140 } 4141 /* 4142 * If a call returns without error, it must have seen UNTRAIN_RET. 4143 * Therefore any non-error return is a success. 4144 */ 4145 return 0; 4146 4147 case INSN_RETURN: 4148 WARN_INSN(insn, "RET before UNTRAIN"); 4149 return 1; 4150 4151 case INSN_SYSCALL: 4152 break; 4153 4154 case INSN_SYSRET: 4155 return 0; 4156 4157 case INSN_NOP: 4158 if (insn->retpoline_safe) 4159 return 0; 4160 break; 4161 4162 default: 4163 break; 4164 } 4165 4166 if (insn->dead_end) 4167 return 0; 4168 4169 if (!next) { 4170 WARN_INSN(insn, "teh end!"); 4171 return 1; 4172 } 4173 insn = next; 4174 } 4175 4176 return 0; 4177 } 4178 4179 /* 4180 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter 4181 * VALIDATE_UNRET_END before RET. 4182 */ 4183 static int validate_unrets(struct objtool_file *file) 4184 { 4185 struct instruction *insn; 4186 int warnings = 0; 4187 4188 for_each_insn(file, insn) { 4189 if (!insn->unret) 4190 continue; 4191 4192 warnings += validate_unret(file, insn); 4193 } 4194 4195 return warnings; 4196 } 4197 4198 static int validate_retpoline(struct objtool_file *file) 4199 { 4200 struct instruction *insn; 4201 int warnings = 0; 4202 4203 for_each_insn(file, insn) { 4204 if (insn->type != INSN_JUMP_DYNAMIC && 4205 insn->type != INSN_CALL_DYNAMIC && 4206 insn->type != INSN_RETURN) 4207 continue; 4208 4209 if (insn->retpoline_safe) 4210 continue; 4211 4212 if (insn->sec->init) 4213 continue; 4214 4215 if (insn->type == INSN_RETURN) { 4216 if (opts.rethunk) { 4217 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build"); 4218 warnings++; 4219 } 4220 continue; 4221 } 4222 4223 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build", 4224 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); 4225 warnings++; 4226 } 4227 4228 if (!opts.cfi) 4229 return warnings; 4230 4231 /* 4232 * kCFI call sites look like: 4233 * 4234 * movl $(-0x12345678), %r10d 4235 * addl -4(%r11), %r10d 4236 * jz 1f 4237 * ud2 4238 * 1: cs call __x86_indirect_thunk_r11 4239 * 4240 * Verify all indirect calls are kCFI adorned by checking for the 4241 * UD2. Notably, doing __nocfi calls to regular (cfi) functions is 4242 * broken. 4243 */ 4244 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 4245 struct symbol *sym = insn->sym; 4246 4247 if (sym && (sym->type == STT_NOTYPE || 4248 sym->type == STT_FUNC) && !sym->nocfi) { 4249 struct instruction *prev = 4250 prev_insn_same_sym(file, insn); 4251 4252 if (!prev || prev->type != INSN_BUG) { 4253 WARN_INSN(insn, "no-cfi indirect call!"); 4254 warnings++; 4255 } 4256 } 4257 } 4258 4259 return warnings; 4260 } 4261 4262 static bool is_kasan_insn(struct instruction *insn) 4263 { 4264 return (insn->type == INSN_CALL && 4265 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return")); 4266 } 4267 4268 static bool is_ubsan_insn(struct instruction *insn) 4269 { 4270 return (insn->type == INSN_CALL && 4271 !strcmp(insn_call_dest(insn)->name, 4272 "__ubsan_handle_builtin_unreachable")); 4273 } 4274 4275 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) 4276 { 4277 struct symbol *func = insn_func(insn); 4278 struct instruction *prev_insn; 4279 int i; 4280 4281 if (insn->type == INSN_NOP || insn->type == INSN_TRAP || 4282 insn->hole || (func && func->ignore)) 4283 return true; 4284 4285 /* 4286 * Ignore alternative replacement instructions. This can happen 4287 * when a whitelisted function uses one of the ALTERNATIVE macros. 4288 */ 4289 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 4290 !strcmp(insn->sec->name, ".altinstr_aux")) 4291 return true; 4292 4293 if (!func) 4294 return false; 4295 4296 if (func->static_call_tramp) 4297 return true; 4298 4299 /* 4300 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees 4301 * __builtin_unreachable(). The BUG() macro has an unreachable() after 4302 * the UD2, which causes GCC's undefined trap logic to emit another UD2 4303 * (or occasionally a JMP to UD2). 4304 * 4305 * It may also insert a UD2 after calling a __noreturn function. 4306 */ 4307 prev_insn = prev_insn_same_sec(file, insn); 4308 if (prev_insn && prev_insn->dead_end && 4309 (insn->type == INSN_BUG || 4310 (insn->type == INSN_JUMP_UNCONDITIONAL && 4311 insn->jump_dest && insn->jump_dest->type == INSN_BUG))) 4312 return true; 4313 4314 /* 4315 * Check if this (or a subsequent) instruction is related to 4316 * CONFIG_UBSAN or CONFIG_KASAN. 4317 * 4318 * End the search at 5 instructions to avoid going into the weeds. 4319 */ 4320 for (i = 0; i < 5; i++) { 4321 4322 if (is_kasan_insn(insn) || is_ubsan_insn(insn)) 4323 return true; 4324 4325 if (insn->type == INSN_JUMP_UNCONDITIONAL) { 4326 if (insn->jump_dest && 4327 insn_func(insn->jump_dest) == func) { 4328 insn = insn->jump_dest; 4329 continue; 4330 } 4331 4332 break; 4333 } 4334 4335 if (insn->offset + insn->len >= func->offset + func->len) 4336 break; 4337 4338 insn = next_insn_same_sec(file, insn); 4339 } 4340 4341 return false; 4342 } 4343 4344 /* 4345 * For FineIBT or kCFI, a certain number of bytes preceding the function may be 4346 * NOPs. Those NOPs may be rewritten at runtime and executed, so give them a 4347 * proper function name: __pfx_<func>. 4348 * 4349 * The NOPs may not exist for the following cases: 4350 * 4351 * - compiler cloned functions (*.cold, *.part0, etc) 4352 * - asm functions created with inline asm or without SYM_FUNC_START() 4353 * 4354 * Also, the function may already have a prefix from a previous objtool run 4355 * (livepatch extracted functions, or manually running objtool multiple times). 4356 * 4357 * So return 0 if the NOPs are missing or the function already has a prefix 4358 * symbol. 4359 */ 4360 static int create_prefix_symbol(struct objtool_file *file, struct symbol *func) 4361 { 4362 struct instruction *insn, *prev; 4363 char name[SYM_NAME_LEN]; 4364 struct cfi_state *cfi; 4365 4366 if (!is_func_sym(func) || is_prefix_func(func) || 4367 func->cold || func->static_call_tramp) 4368 return 0; 4369 4370 if ((strlen(func->name) + sizeof("__pfx_") > SYM_NAME_LEN)) { 4371 WARN("%s: symbol name too long, can't create __pfx_ symbol", 4372 func->name); 4373 return 0; 4374 } 4375 4376 if (snprintf_check(name, SYM_NAME_LEN, "__pfx_%s", func->name)) 4377 return -1; 4378 4379 if (file->klp) { 4380 struct symbol *pfx; 4381 4382 pfx = find_symbol_by_offset(func->sec, func->offset - opts.prefix); 4383 if (pfx && is_prefix_func(pfx) && !strcmp(pfx->name, name)) 4384 return 0; 4385 } 4386 4387 insn = find_insn(file, func->sec, func->offset); 4388 if (!insn) { 4389 WARN("%s: can't find starting instruction", func->name); 4390 return -1; 4391 } 4392 4393 for (prev = prev_insn_same_sec(file, insn); 4394 prev; 4395 prev = prev_insn_same_sec(file, prev)) { 4396 u64 offset; 4397 4398 if (prev->type != INSN_NOP) 4399 return 0; 4400 4401 offset = func->offset - prev->offset; 4402 4403 if (offset > opts.prefix) 4404 return 0; 4405 4406 if (offset < opts.prefix) 4407 continue; 4408 4409 if (!elf_create_symbol(file->elf, name, func->sec, 4410 GELF_ST_BIND(func->sym.st_info), 4411 GELF_ST_TYPE(func->sym.st_info), 4412 prev->offset, opts.prefix)) 4413 return -1; 4414 4415 break; 4416 } 4417 4418 if (!prev) 4419 return 0; 4420 4421 if (!insn->cfi) { 4422 /* 4423 * This can happen if stack validation isn't enabled or the 4424 * function is annotated with STACK_FRAME_NON_STANDARD. 4425 */ 4426 return 0; 4427 } 4428 4429 /* Propagate insn->cfi to the prefix code */ 4430 cfi = cfi_hash_find_or_add(insn->cfi); 4431 for (; prev != insn; prev = next_insn_same_sec(file, prev)) 4432 prev->cfi = cfi; 4433 4434 return 0; 4435 } 4436 4437 static int create_prefix_symbols(struct objtool_file *file) 4438 { 4439 struct section *sec; 4440 struct symbol *func; 4441 4442 for_each_sec(file->elf, sec) { 4443 if (!is_text_sec(sec)) 4444 continue; 4445 4446 sec_for_each_sym(sec, func) { 4447 if (create_prefix_symbol(file, func)) 4448 return -1; 4449 } 4450 } 4451 4452 return 0; 4453 } 4454 4455 static int validate_symbol(struct objtool_file *file, struct section *sec, 4456 struct symbol *sym, struct insn_state *state) 4457 { 4458 struct instruction *insn; 4459 struct symbol *func; 4460 int ret; 4461 4462 if (!sym->len) { 4463 WARN("%s() is missing an ELF size annotation", sym->name); 4464 return 1; 4465 } 4466 4467 if (sym->pfunc != sym || sym->alias != sym) 4468 return 0; 4469 4470 insn = find_insn(file, sec, sym->offset); 4471 if (!insn || insn->visited) 4472 return 0; 4473 4474 if (opts.uaccess) 4475 state->uaccess = sym->uaccess_safe; 4476 4477 func = insn_func(insn); 4478 4479 if (opts.checksum) 4480 checksum_init(func); 4481 4482 if (opts.trace && !fnmatch(opts.trace, sym->name, 0)) { 4483 trace_enable(); 4484 TRACE("%s: validation begin\n", sym->name); 4485 } 4486 4487 ret = validate_branch(file, func, insn, *state); 4488 if (ret) 4489 BT_INSN(insn, "<=== (sym)"); 4490 4491 TRACE("%s: validation %s\n\n", sym->name, ret ? "failed" : "end"); 4492 trace_disable(); 4493 4494 if (opts.checksum) 4495 checksum_finish(func); 4496 4497 return ret; 4498 } 4499 4500 static int validate_section(struct objtool_file *file, struct section *sec) 4501 { 4502 struct insn_state state; 4503 struct symbol *func; 4504 int warnings = 0; 4505 4506 sec_for_each_sym(sec, func) { 4507 if (!is_func_sym(func)) 4508 continue; 4509 4510 init_insn_state(file, &state, sec); 4511 set_func_state(&state.cfi); 4512 4513 warnings += validate_symbol(file, sec, func, &state); 4514 } 4515 4516 return warnings; 4517 } 4518 4519 static int validate_noinstr_sections(struct objtool_file *file) 4520 { 4521 struct section *sec; 4522 int warnings = 0; 4523 4524 sec = find_section_by_name(file->elf, ".noinstr.text"); 4525 if (sec) { 4526 warnings += validate_section(file, sec); 4527 warnings += validate_unwind_hints(file, sec); 4528 } 4529 4530 sec = find_section_by_name(file->elf, ".entry.text"); 4531 if (sec) { 4532 warnings += validate_section(file, sec); 4533 warnings += validate_unwind_hints(file, sec); 4534 } 4535 4536 sec = find_section_by_name(file->elf, ".cpuidle.text"); 4537 if (sec) { 4538 warnings += validate_section(file, sec); 4539 warnings += validate_unwind_hints(file, sec); 4540 } 4541 4542 return warnings; 4543 } 4544 4545 static int validate_functions(struct objtool_file *file) 4546 { 4547 struct section *sec; 4548 int warnings = 0; 4549 4550 for_each_sec(file->elf, sec) { 4551 if (!is_text_sec(sec)) 4552 continue; 4553 4554 warnings += validate_section(file, sec); 4555 } 4556 4557 return warnings; 4558 } 4559 4560 static void mark_endbr_used(struct instruction *insn) 4561 { 4562 if (!list_empty(&insn->call_node)) 4563 list_del_init(&insn->call_node); 4564 } 4565 4566 static bool noendbr_range(struct objtool_file *file, struct instruction *insn) 4567 { 4568 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1); 4569 struct instruction *first; 4570 4571 if (!sym) 4572 return false; 4573 4574 first = find_insn(file, sym->sec, sym->offset); 4575 if (!first) 4576 return false; 4577 4578 if (first->type != INSN_ENDBR && !first->noendbr) 4579 return false; 4580 4581 return insn->offset == sym->offset + sym->len; 4582 } 4583 4584 static int __validate_ibt_insn(struct objtool_file *file, struct instruction *insn, 4585 struct instruction *dest) 4586 { 4587 if (dest->type == INSN_ENDBR) { 4588 mark_endbr_used(dest); 4589 return 0; 4590 } 4591 4592 if (insn_func(dest) && insn_func(insn) && 4593 insn_func(dest)->pfunc == insn_func(insn)->pfunc) { 4594 /* 4595 * Anything from->to self is either _THIS_IP_ or 4596 * IRET-to-self. 4597 * 4598 * There is no sane way to annotate _THIS_IP_ since the 4599 * compiler treats the relocation as a constant and is 4600 * happy to fold in offsets, skewing any annotation we 4601 * do, leading to vast amounts of false-positives. 4602 * 4603 * There's also compiler generated _THIS_IP_ through 4604 * KCOV and such which we have no hope of annotating. 4605 * 4606 * As such, blanket accept self-references without 4607 * issue. 4608 */ 4609 return 0; 4610 } 4611 4612 /* 4613 * Accept anything ANNOTATE_NOENDBR. 4614 */ 4615 if (dest->noendbr) 4616 return 0; 4617 4618 /* 4619 * Accept if this is the instruction after a symbol 4620 * that is (no)endbr -- typical code-range usage. 4621 */ 4622 if (noendbr_range(file, dest)) 4623 return 0; 4624 4625 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4626 return 1; 4627 } 4628 4629 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn) 4630 { 4631 struct instruction *dest; 4632 struct reloc *reloc; 4633 unsigned long off; 4634 int warnings = 0; 4635 4636 /* 4637 * Looking for function pointer load relocations. Ignore 4638 * direct/indirect branches: 4639 */ 4640 switch (insn->type) { 4641 4642 case INSN_CALL: 4643 case INSN_CALL_DYNAMIC: 4644 case INSN_JUMP_CONDITIONAL: 4645 case INSN_JUMP_UNCONDITIONAL: 4646 case INSN_JUMP_DYNAMIC: 4647 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4648 case INSN_RETURN: 4649 case INSN_NOP: 4650 return 0; 4651 4652 case INSN_LEA_RIP: 4653 if (!insn_reloc(file, insn)) { 4654 /* local function pointer reference without reloc */ 4655 4656 off = arch_jump_destination(insn); 4657 4658 dest = find_insn(file, insn->sec, off); 4659 if (!dest) { 4660 WARN_INSN(insn, "corrupt function pointer reference"); 4661 return 1; 4662 } 4663 4664 return __validate_ibt_insn(file, insn, dest); 4665 } 4666 break; 4667 4668 default: 4669 break; 4670 } 4671 4672 for (reloc = insn_reloc(file, insn); 4673 reloc; 4674 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 4675 reloc_offset(reloc) + 1, 4676 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) { 4677 4678 off = reloc->sym->offset + arch_insn_adjusted_addend(insn, reloc); 4679 4680 dest = find_insn(file, reloc->sym->sec, off); 4681 if (!dest) 4682 continue; 4683 4684 warnings += __validate_ibt_insn(file, insn, dest); 4685 } 4686 4687 return warnings; 4688 } 4689 4690 static int validate_ibt_data_reloc(struct objtool_file *file, 4691 struct reloc *reloc) 4692 { 4693 struct instruction *dest; 4694 4695 dest = find_insn(file, reloc->sym->sec, 4696 reloc->sym->offset + reloc_addend(reloc)); 4697 if (!dest) 4698 return 0; 4699 4700 if (dest->type == INSN_ENDBR) { 4701 mark_endbr_used(dest); 4702 return 0; 4703 } 4704 4705 if (dest->noendbr) 4706 return 0; 4707 4708 WARN_FUNC(reloc->sec->base, reloc_offset(reloc), 4709 "data relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4710 4711 return 1; 4712 } 4713 4714 /* 4715 * Validate IBT rules and remove used ENDBR instructions from the seal list. 4716 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with 4717 * NOPs) later, in create_ibt_endbr_seal_sections(). 4718 */ 4719 static int validate_ibt(struct objtool_file *file) 4720 { 4721 struct section *sec; 4722 struct reloc *reloc; 4723 struct instruction *insn; 4724 int warnings = 0; 4725 4726 for_each_insn(file, insn) 4727 warnings += validate_ibt_insn(file, insn); 4728 4729 for_each_sec(file->elf, sec) { 4730 4731 /* Already done by validate_ibt_insn() */ 4732 if (is_text_sec(sec)) 4733 continue; 4734 4735 if (!sec->rsec) 4736 continue; 4737 4738 /* 4739 * These sections can reference text addresses, but not with 4740 * the intent to indirect branch to them. 4741 */ 4742 if ((!strncmp(sec->name, ".discard", 8) && 4743 strcmp(sec->name, ".discard.ibt_endbr_noseal")) || 4744 !strncmp(sec->name, ".debug", 6) || 4745 !strcmp(sec->name, ".altinstructions") || 4746 !strcmp(sec->name, ".ibt_endbr_seal") || 4747 !strcmp(sec->name, ".kcfi_traps") || 4748 !strcmp(sec->name, ".orc_unwind_ip") || 4749 !strcmp(sec->name, ".retpoline_sites") || 4750 !strcmp(sec->name, ".smp_locks") || 4751 !strcmp(sec->name, ".static_call_sites") || 4752 !strcmp(sec->name, "_error_injection_whitelist") || 4753 !strcmp(sec->name, "_kprobe_blacklist") || 4754 !strcmp(sec->name, "__bug_table") || 4755 !strcmp(sec->name, "__ex_table") || 4756 !strcmp(sec->name, "__jump_table") || 4757 !strcmp(sec->name, "__klp_funcs") || 4758 !strcmp(sec->name, "__mcount_loc") || 4759 !strcmp(sec->name, ".llvm.call-graph-profile") || 4760 !strcmp(sec->name, ".llvm_bb_addr_map") || 4761 !strcmp(sec->name, "__tracepoints") || 4762 !strcmp(sec->name, "__patchable_function_entries")) 4763 continue; 4764 4765 for_each_reloc(sec->rsec, reloc) 4766 warnings += validate_ibt_data_reloc(file, reloc); 4767 } 4768 4769 return warnings; 4770 } 4771 4772 static int validate_sls(struct objtool_file *file) 4773 { 4774 struct instruction *insn, *next_insn; 4775 int warnings = 0; 4776 4777 for_each_insn(file, insn) { 4778 next_insn = next_insn_same_sec(file, insn); 4779 4780 if (insn->retpoline_safe) 4781 continue; 4782 4783 switch (insn->type) { 4784 case INSN_RETURN: 4785 if (!next_insn || next_insn->type != INSN_TRAP) { 4786 WARN_INSN(insn, "missing int3 after ret"); 4787 warnings++; 4788 } 4789 4790 break; 4791 case INSN_JUMP_DYNAMIC: 4792 if (!next_insn || next_insn->type != INSN_TRAP) { 4793 WARN_INSN(insn, "missing int3 after indirect jump"); 4794 warnings++; 4795 } 4796 break; 4797 default: 4798 break; 4799 } 4800 } 4801 4802 return warnings; 4803 } 4804 4805 static int validate_reachable_instructions(struct objtool_file *file) 4806 { 4807 struct instruction *insn, *prev_insn; 4808 struct symbol *call_dest; 4809 int warnings = 0; 4810 4811 if (file->ignore_unreachables) 4812 return 0; 4813 4814 for_each_insn(file, insn) { 4815 if (insn->visited || ignore_unreachable_insn(file, insn)) 4816 continue; 4817 4818 prev_insn = prev_insn_same_sec(file, insn); 4819 if (prev_insn && prev_insn->dead_end) { 4820 call_dest = insn_call_dest(prev_insn); 4821 if (call_dest) { 4822 WARN_INSN(insn, "%s() missing __noreturn in .c/.h or NORETURN() in noreturns.h", 4823 call_dest->name); 4824 warnings++; 4825 continue; 4826 } 4827 } 4828 4829 WARN_INSN(insn, "unreachable instruction"); 4830 warnings++; 4831 } 4832 4833 return warnings; 4834 } 4835 4836 __weak bool arch_absolute_reloc(struct elf *elf, struct reloc *reloc) 4837 { 4838 unsigned int type = reloc_type(reloc); 4839 size_t sz = elf_addr_size(elf); 4840 4841 return (sz == 8) ? (type == R_ABS64) : (type == R_ABS32); 4842 } 4843 4844 static int check_abs_references(struct objtool_file *file) 4845 { 4846 struct section *sec; 4847 struct reloc *reloc; 4848 int ret = 0; 4849 4850 for_each_sec(file->elf, sec) { 4851 /* absolute references in non-loadable sections are fine */ 4852 if (!(sec->sh.sh_flags & SHF_ALLOC)) 4853 continue; 4854 4855 /* section must have an associated .rela section */ 4856 if (!sec->rsec) 4857 continue; 4858 4859 /* 4860 * Special case for compiler generated metadata that is not 4861 * consumed until after boot. 4862 */ 4863 if (!strcmp(sec->name, "__patchable_function_entries")) 4864 continue; 4865 4866 for_each_reloc(sec->rsec, reloc) { 4867 if (arch_absolute_reloc(file->elf, reloc)) { 4868 WARN("section %s has absolute relocation at offset 0x%llx", 4869 sec->name, (unsigned long long)reloc_offset(reloc)); 4870 ret++; 4871 } 4872 } 4873 } 4874 return ret; 4875 } 4876 4877 struct insn_chunk { 4878 void *addr; 4879 struct insn_chunk *next; 4880 }; 4881 4882 /* 4883 * Reduce peak RSS usage by freeing insns memory before writing the ELF file, 4884 * which can trigger more allocations for .debug_* sections whose data hasn't 4885 * been read yet. 4886 */ 4887 static void free_insns(struct objtool_file *file) 4888 { 4889 struct instruction *insn; 4890 struct insn_chunk *chunks = NULL, *chunk; 4891 4892 for_each_insn(file, insn) { 4893 if (!insn->idx) { 4894 chunk = malloc(sizeof(*chunk)); 4895 chunk->addr = insn; 4896 chunk->next = chunks; 4897 chunks = chunk; 4898 } 4899 } 4900 4901 for (chunk = chunks; chunk; chunk = chunk->next) 4902 free(chunk->addr); 4903 } 4904 4905 const char *objtool_disas_insn(struct instruction *insn) 4906 { 4907 struct disas_context *dctx = objtool_disas_ctx; 4908 4909 if (!dctx) 4910 return ""; 4911 4912 disas_insn(dctx, insn); 4913 return disas_result(dctx); 4914 } 4915 4916 int check(struct objtool_file *file) 4917 { 4918 struct disas_context *disas_ctx = NULL; 4919 int ret = 0, warnings = 0; 4920 4921 /* 4922 * If the verbose or backtrace option is used then we need a 4923 * disassembly context to disassemble instruction or function 4924 * on warning or backtrace. 4925 */ 4926 if (opts.verbose || opts.backtrace || opts.trace) { 4927 disas_ctx = disas_context_create(file); 4928 if (!disas_ctx) 4929 opts.trace = false; 4930 objtool_disas_ctx = disas_ctx; 4931 } 4932 4933 arch_initial_func_cfi_state(&initial_func_cfi); 4934 init_cfi_state(&init_cfi); 4935 init_cfi_state(&func_cfi); 4936 set_func_state(&func_cfi); 4937 init_cfi_state(&force_undefined_cfi); 4938 force_undefined_cfi.force_undefined = true; 4939 4940 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) { 4941 ret = -1; 4942 goto out; 4943 } 4944 4945 cfi_hash_add(&init_cfi); 4946 cfi_hash_add(&func_cfi); 4947 4948 ret = checksum_debug_init(file); 4949 if (ret) 4950 goto out; 4951 4952 ret = decode_sections(file); 4953 if (ret) 4954 goto out; 4955 4956 if (!nr_insns) 4957 goto out; 4958 4959 if (opts.retpoline) 4960 warnings += validate_retpoline(file); 4961 4962 if (validate_branch_enabled()) { 4963 int w = 0; 4964 4965 w += validate_functions(file); 4966 w += validate_unwind_hints(file, NULL); 4967 if (!w) 4968 w += validate_reachable_instructions(file); 4969 4970 warnings += w; 4971 4972 } else if (opts.noinstr) { 4973 warnings += validate_noinstr_sections(file); 4974 } 4975 4976 if (opts.unret) { 4977 /* 4978 * Must be after validate_branch() and friends, it plays 4979 * further games with insn->visited. 4980 */ 4981 warnings += validate_unrets(file); 4982 } 4983 4984 if (opts.ibt) 4985 warnings += validate_ibt(file); 4986 4987 if (opts.sls) 4988 warnings += validate_sls(file); 4989 4990 if (opts.static_call) { 4991 ret = create_static_call_sections(file); 4992 if (ret) 4993 goto out; 4994 } 4995 4996 if (opts.retpoline) { 4997 ret = create_retpoline_sites_sections(file); 4998 if (ret) 4999 goto out; 5000 } 5001 5002 if (opts.cfi) { 5003 ret = create_cfi_sections(file); 5004 if (ret) 5005 goto out; 5006 } 5007 5008 if (opts.rethunk) { 5009 ret = create_return_sites_sections(file); 5010 if (ret) 5011 goto out; 5012 5013 if (opts.hack_skylake) { 5014 ret = create_direct_call_sections(file); 5015 if (ret) 5016 goto out; 5017 } 5018 } 5019 5020 if (opts.mcount) { 5021 ret = create_mcount_loc_sections(file); 5022 if (ret) 5023 goto out; 5024 } 5025 5026 if (opts.prefix) { 5027 ret = create_prefix_symbols(file); 5028 if (ret) 5029 goto out; 5030 } 5031 5032 if (opts.ibt) { 5033 ret = create_ibt_endbr_seal_sections(file); 5034 if (ret) 5035 goto out; 5036 } 5037 5038 if (opts.noabs) 5039 warnings += check_abs_references(file); 5040 5041 if (opts.checksum) { 5042 ret = create_sym_checksum_section(file); 5043 if (ret) 5044 goto out; 5045 } 5046 5047 if (opts.orc && nr_insns) { 5048 ret = orc_create(file); 5049 if (ret) 5050 goto out; 5051 } 5052 5053 if (opts.stats) { 5054 printf("nr_insns_visited: %ld\n", nr_insns_visited); 5055 printf("nr_cfi: %ld\n", nr_cfi); 5056 printf("nr_cfi_reused: %ld\n", nr_cfi_reused); 5057 printf("nr_cfi_cache: %ld\n", nr_cfi_cache); 5058 } 5059 5060 out: 5061 if (!ret && !warnings) { 5062 free_insns(file); 5063 return 0; 5064 } 5065 5066 if (opts.werror && warnings) 5067 ret = 1; 5068 5069 if (opts.verbose) { 5070 if (opts.werror && warnings) 5071 WARN("%d warning(s) upgraded to errors", warnings); 5072 disas_warned_funcs(disas_ctx); 5073 } 5074 5075 if (disas_ctx) { 5076 disas_context_destroy(disas_ctx); 5077 objtool_disas_ctx = NULL; 5078 } 5079 5080 free_insns(file); 5081 5082 if (opts.backup && make_backup()) 5083 return 1; 5084 5085 return ret; 5086 } 5087