1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6 #define _GNU_SOURCE /* memmem() */ 7 #include <string.h> 8 #include <stdlib.h> 9 #include <inttypes.h> 10 #include <sys/mman.h> 11 12 #include <objtool/builtin.h> 13 #include <objtool/cfi.h> 14 #include <objtool/arch.h> 15 #include <objtool/disas.h> 16 #include <objtool/check.h> 17 #include <objtool/special.h> 18 #include <objtool/warn.h> 19 #include <objtool/checksum.h> 20 #include <objtool/util.h> 21 22 #include <linux/objtool_types.h> 23 #include <linux/hashtable.h> 24 #include <linux/kernel.h> 25 #include <linux/static_call_types.h> 26 #include <linux/string.h> 27 28 struct alternative { 29 struct alternative *next; 30 struct instruction *insn; 31 }; 32 33 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 34 35 static struct cfi_init_state initial_func_cfi; 36 static struct cfi_state init_cfi; 37 static struct cfi_state func_cfi; 38 static struct cfi_state force_undefined_cfi; 39 40 struct instruction *find_insn(struct objtool_file *file, 41 struct section *sec, unsigned long offset) 42 { 43 struct instruction *insn; 44 45 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) { 46 if (insn->sec == sec && insn->offset == offset) 47 return insn; 48 } 49 50 return NULL; 51 } 52 53 struct instruction *next_insn_same_sec(struct objtool_file *file, 54 struct instruction *insn) 55 { 56 if (insn->idx == INSN_CHUNK_MAX) 57 return find_insn(file, insn->sec, insn->offset + insn->len); 58 59 insn++; 60 if (!insn->len) 61 return NULL; 62 63 return insn; 64 } 65 66 static struct instruction *next_insn_same_func(struct objtool_file *file, 67 struct instruction *insn) 68 { 69 struct instruction *next = next_insn_same_sec(file, insn); 70 struct symbol *func = insn_func(insn); 71 72 if (!func) 73 return NULL; 74 75 if (next && insn_func(next) == func) 76 return next; 77 78 /* Check if we're already in the subfunction: */ 79 if (func == func->cfunc) 80 return NULL; 81 82 /* Move to the subfunction: */ 83 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 84 } 85 86 static struct instruction *prev_insn_same_sec(struct objtool_file *file, 87 struct instruction *insn) 88 { 89 if (insn->idx == 0) { 90 if (insn->prev_len) 91 return find_insn(file, insn->sec, insn->offset - insn->prev_len); 92 return NULL; 93 } 94 95 return insn - 1; 96 } 97 98 static struct instruction *prev_insn_same_sym(struct objtool_file *file, 99 struct instruction *insn) 100 { 101 struct instruction *prev = prev_insn_same_sec(file, insn); 102 103 if (prev && insn_func(prev) == insn_func(insn)) 104 return prev; 105 106 return NULL; 107 } 108 109 #define for_each_insn(file, insn) \ 110 for (struct section *__sec, *__fake = (struct section *)1; \ 111 __fake; __fake = NULL) \ 112 for_each_sec(file->elf, __sec) \ 113 sec_for_each_insn(file, __sec, insn) 114 115 #define func_for_each_insn(file, func, insn) \ 116 for (insn = find_insn(file, func->sec, func->offset); \ 117 insn; \ 118 insn = next_insn_same_func(file, insn)) 119 120 #define sym_for_each_insn(file, sym, insn) \ 121 for (insn = find_insn(file, sym->sec, sym->offset); \ 122 insn && insn->offset < sym->offset + sym->len; \ 123 insn = next_insn_same_sec(file, insn)) 124 125 #define sym_for_each_insn_continue_reverse(file, sym, insn) \ 126 for (insn = prev_insn_same_sec(file, insn); \ 127 insn && insn->offset >= sym->offset; \ 128 insn = prev_insn_same_sec(file, insn)) 129 130 #define sec_for_each_insn_from(file, insn) \ 131 for (; insn; insn = next_insn_same_sec(file, insn)) 132 133 #define sec_for_each_insn_continue(file, insn) \ 134 for (insn = next_insn_same_sec(file, insn); insn; \ 135 insn = next_insn_same_sec(file, insn)) 136 137 static inline struct symbol *insn_call_dest(struct instruction *insn) 138 { 139 if (insn->type == INSN_JUMP_DYNAMIC || 140 insn->type == INSN_CALL_DYNAMIC) 141 return NULL; 142 143 return insn->_call_dest; 144 } 145 146 static inline struct reloc *insn_jump_table(struct instruction *insn) 147 { 148 if (insn->type == INSN_JUMP_DYNAMIC || 149 insn->type == INSN_CALL_DYNAMIC) 150 return insn->_jump_table; 151 152 return NULL; 153 } 154 155 static inline unsigned long insn_jump_table_size(struct instruction *insn) 156 { 157 if (insn->type == INSN_JUMP_DYNAMIC || 158 insn->type == INSN_CALL_DYNAMIC) 159 return insn->_jump_table_size; 160 161 return 0; 162 } 163 164 static bool is_jump_table_jump(struct instruction *insn) 165 { 166 struct alt_group *alt_group = insn->alt_group; 167 168 if (insn_jump_table(insn)) 169 return true; 170 171 /* Retpoline alternative for a jump table? */ 172 return alt_group && alt_group->orig_group && 173 insn_jump_table(alt_group->orig_group->first_insn); 174 } 175 176 static bool is_sibling_call(struct instruction *insn) 177 { 178 /* 179 * Assume only STT_FUNC calls have jump-tables. 180 */ 181 if (insn_func(insn)) { 182 /* An indirect jump is either a sibling call or a jump to a table. */ 183 if (insn->type == INSN_JUMP_DYNAMIC) 184 return !is_jump_table_jump(insn); 185 } 186 187 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */ 188 return (is_static_jump(insn) && insn_call_dest(insn)); 189 } 190 191 /* 192 * Checks if a function is a Rust "noreturn" one. 193 */ 194 static bool is_rust_noreturn(const struct symbol *func) 195 { 196 /* 197 * If it does not start with "_R", then it is not a Rust symbol. 198 */ 199 if (strncmp(func->name, "_R", 2)) 200 return false; 201 202 /* 203 * These are just heuristics -- we do not control the precise symbol 204 * name, due to the crate disambiguators (which depend on the compiler) 205 * as well as changes to the source code itself between versions (since 206 * these come from the Rust standard library). 207 */ 208 return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || 209 str_ends_with(func->name, "_4core6option13expect_failed") || 210 str_ends_with(func->name, "_4core6option13unwrap_failed") || 211 str_ends_with(func->name, "_4core6result13unwrap_failed") || 212 str_ends_with(func->name, "_4core9panicking5panic") || 213 str_ends_with(func->name, "_4core9panicking9panic_fmt") || 214 str_ends_with(func->name, "_4core9panicking14panic_explicit") || 215 str_ends_with(func->name, "_4core9panicking14panic_nounwind") || 216 str_ends_with(func->name, "_4core9panicking18panic_bounds_check") || 217 str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt") || 218 str_ends_with(func->name, "_4core9panicking19assert_failed_inner") || 219 str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference") || 220 str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") || 221 str_ends_with(func->name, "_7___rustc17rust_begin_unwind") || 222 strstr(func->name, "_4core9panicking13assert_failed") || 223 strstr(func->name, "_4core9panicking11panic_const24panic_const_") || 224 (strstr(func->name, "_4core5slice5index") && 225 strstr(func->name, "slice_") && 226 str_ends_with(func->name, "_fail")); 227 } 228 229 /* 230 * This checks to see if the given function is a "noreturn" function. 231 * 232 * For global functions which are outside the scope of this object file, we 233 * have to keep a manual list of them. 234 * 235 * For local functions, we have to detect them manually by simply looking for 236 * the lack of a return instruction. 237 */ 238 static bool __dead_end_function(struct objtool_file *file, struct symbol *func, 239 int recursion) 240 { 241 int i; 242 struct instruction *insn; 243 bool empty = true; 244 245 #define NORETURN(func) __stringify(func), 246 static const char * const global_noreturns[] = { 247 #include "noreturns.h" 248 }; 249 #undef NORETURN 250 251 if (!func) 252 return false; 253 254 if (!is_local_sym(func)) { 255 if (is_rust_noreturn(func)) 256 return true; 257 258 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) 259 if (!strcmp(func->name, global_noreturns[i])) 260 return true; 261 } 262 263 if (is_weak_sym(func)) 264 return false; 265 266 if (!func->len) 267 return false; 268 269 insn = find_insn(file, func->sec, func->offset); 270 if (!insn || !insn_func(insn)) 271 return false; 272 273 func_for_each_insn(file, func, insn) { 274 empty = false; 275 276 if (insn->type == INSN_RETURN) 277 return false; 278 } 279 280 if (empty) 281 return false; 282 283 /* 284 * A function can have a sibling call instead of a return. In that 285 * case, the function's dead-end status depends on whether the target 286 * of the sibling call returns. 287 */ 288 func_for_each_insn(file, func, insn) { 289 if (is_sibling_call(insn)) { 290 struct instruction *dest = insn->jump_dest; 291 292 if (!dest) 293 /* sibling call to another file */ 294 return false; 295 296 /* local sibling call */ 297 if (recursion == 5) { 298 /* 299 * Infinite recursion: two functions have 300 * sibling calls to each other. This is a very 301 * rare case. It means they aren't dead ends. 302 */ 303 return false; 304 } 305 306 return __dead_end_function(file, insn_func(dest), recursion+1); 307 } 308 } 309 310 return true; 311 } 312 313 static bool dead_end_function(struct objtool_file *file, struct symbol *func) 314 { 315 return __dead_end_function(file, func, 0); 316 } 317 318 static void init_cfi_state(struct cfi_state *cfi) 319 { 320 int i; 321 322 for (i = 0; i < CFI_NUM_REGS; i++) { 323 cfi->regs[i].base = CFI_UNDEFINED; 324 cfi->vals[i].base = CFI_UNDEFINED; 325 } 326 cfi->cfa.base = CFI_UNDEFINED; 327 cfi->drap_reg = CFI_UNDEFINED; 328 cfi->drap_offset = -1; 329 } 330 331 static void init_insn_state(struct objtool_file *file, struct insn_state *state, 332 struct section *sec) 333 { 334 memset(state, 0, sizeof(*state)); 335 init_cfi_state(&state->cfi); 336 337 if (opts.noinstr && sec) 338 state->noinstr = sec->noinstr; 339 } 340 341 static struct cfi_state *cfi_alloc(void) 342 { 343 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state)); 344 if (!cfi) { 345 ERROR_GLIBC("calloc"); 346 exit(1); 347 } 348 nr_cfi++; 349 return cfi; 350 } 351 352 static int cfi_bits; 353 static struct hlist_head *cfi_hash; 354 355 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2) 356 { 357 return memcmp((void *)cfi1 + sizeof(cfi1->hash), 358 (void *)cfi2 + sizeof(cfi2->hash), 359 sizeof(struct cfi_state) - sizeof(struct hlist_node)); 360 } 361 362 static inline u32 cfi_key(struct cfi_state *cfi) 363 { 364 return jhash((void *)cfi + sizeof(cfi->hash), 365 sizeof(*cfi) - sizeof(cfi->hash), 0); 366 } 367 368 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi) 369 { 370 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 371 struct cfi_state *obj; 372 373 hlist_for_each_entry(obj, head, hash) { 374 if (!cficmp(cfi, obj)) { 375 nr_cfi_cache++; 376 return obj; 377 } 378 } 379 380 obj = cfi_alloc(); 381 *obj = *cfi; 382 hlist_add_head(&obj->hash, head); 383 384 return obj; 385 } 386 387 static void cfi_hash_add(struct cfi_state *cfi) 388 { 389 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 390 391 hlist_add_head(&cfi->hash, head); 392 } 393 394 static void *cfi_hash_alloc(unsigned long size) 395 { 396 cfi_bits = max(10, ilog2(size)); 397 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits, 398 PROT_READ|PROT_WRITE, 399 MAP_PRIVATE|MAP_ANON, -1, 0); 400 if (cfi_hash == (void *)-1L) { 401 ERROR_GLIBC("mmap fail cfi_hash"); 402 cfi_hash = NULL; 403 } else if (opts.stats) { 404 printf("cfi_bits: %d\n", cfi_bits); 405 } 406 407 return cfi_hash; 408 } 409 410 static unsigned long nr_insns; 411 static unsigned long nr_insns_visited; 412 413 /* 414 * Call the arch-specific instruction decoder for all the instructions and add 415 * them to the global instruction list. 416 */ 417 static int decode_instructions(struct objtool_file *file) 418 { 419 struct section *sec; 420 struct symbol *func; 421 unsigned long offset; 422 struct instruction *insn; 423 424 for_each_sec(file->elf, sec) { 425 struct instruction *insns = NULL; 426 u8 prev_len = 0; 427 u8 idx = 0; 428 429 if (!is_text_sec(sec)) 430 continue; 431 432 if (strcmp(sec->name, ".altinstr_replacement") && 433 strcmp(sec->name, ".altinstr_aux") && 434 strncmp(sec->name, ".discard.", 9)) 435 sec->text = true; 436 437 if (!strcmp(sec->name, ".noinstr.text") || 438 !strcmp(sec->name, ".entry.text") || 439 !strcmp(sec->name, ".cpuidle.text") || 440 !strncmp(sec->name, ".text..__x86.", 13)) 441 sec->noinstr = true; 442 443 /* 444 * .init.text code is ran before userspace and thus doesn't 445 * strictly need retpolines, except for modules which are 446 * loaded late, they very much do need retpoline in their 447 * .init.text 448 */ 449 if (!strcmp(sec->name, ".init.text") && !opts.module) 450 sec->init = true; 451 452 for (offset = 0; offset < sec_size(sec); offset += insn->len) { 453 if (!insns || idx == INSN_CHUNK_MAX) { 454 insns = calloc(INSN_CHUNK_SIZE, sizeof(*insn)); 455 if (!insns) { 456 ERROR_GLIBC("calloc"); 457 return -1; 458 } 459 idx = 0; 460 } else { 461 idx++; 462 } 463 insn = &insns[idx]; 464 insn->idx = idx; 465 466 INIT_LIST_HEAD(&insn->call_node); 467 insn->sec = sec; 468 insn->offset = offset; 469 insn->prev_len = prev_len; 470 471 if (arch_decode_instruction(file, sec, offset, sec_size(sec) - offset, insn)) 472 return -1; 473 474 prev_len = insn->len; 475 476 /* 477 * By default, "ud2" is a dead end unless otherwise 478 * annotated, because GCC 7 inserts it for certain 479 * divide-by-zero cases. 480 */ 481 if (insn->type == INSN_BUG) 482 insn->dead_end = true; 483 484 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset)); 485 nr_insns++; 486 } 487 488 sec_for_each_sym(sec, func) { 489 if (!is_notype_sym(func) && !is_func_sym(func)) 490 continue; 491 492 if (func->offset == sec_size(sec)) { 493 /* Heuristic: likely an "end" symbol */ 494 if (is_notype_sym(func)) 495 continue; 496 ERROR("%s(): STT_FUNC at end of section", func->name); 497 return -1; 498 } 499 500 if (func->embedded_insn || func->alias != func) 501 continue; 502 503 if (!find_insn(file, sec, func->offset)) { 504 ERROR("%s(): can't find starting instruction", func->name); 505 return -1; 506 } 507 508 sym_for_each_insn(file, func, insn) { 509 insn->sym = func; 510 if (is_func_sym(func) && 511 insn->type == INSN_ENDBR && 512 list_empty(&insn->call_node)) { 513 if (insn->offset == func->offset) { 514 list_add_tail(&insn->call_node, &file->endbr_list); 515 file->nr_endbr++; 516 } else { 517 file->nr_endbr_int++; 518 } 519 } 520 } 521 } 522 } 523 524 if (opts.stats) 525 printf("nr_insns: %lu\n", nr_insns); 526 527 return 0; 528 } 529 530 /* 531 * Read the pv_ops[] .data table to find the static initialized values. 532 */ 533 static int add_pv_ops(struct objtool_file *file, const char *symname) 534 { 535 struct symbol *sym, *func; 536 unsigned long off, end; 537 struct reloc *reloc; 538 int idx; 539 540 sym = find_symbol_by_name(file->elf, symname); 541 if (!sym) 542 return 0; 543 544 off = sym->offset; 545 end = off + sym->len; 546 for (;;) { 547 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 548 if (!reloc) 549 break; 550 551 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long); 552 553 func = reloc->sym; 554 if (is_sec_sym(func)) 555 func = find_symbol_by_offset(reloc->sym->sec, 556 reloc_addend(reloc)); 557 if (!func) { 558 ERROR_FUNC(reloc->sym->sec, reloc_addend(reloc), 559 "can't find func at %s[%d]", symname, idx); 560 return -1; 561 } 562 563 if (objtool_pv_add(file, idx, func)) 564 return -1; 565 566 off = reloc_offset(reloc) + 1; 567 if (off > end) 568 break; 569 } 570 571 return 0; 572 } 573 574 /* 575 * Allocate and initialize file->pv_ops[]. 576 */ 577 static int init_pv_ops(struct objtool_file *file) 578 { 579 static const char *pv_ops_tables[] = { 580 "pv_ops", 581 "xen_cpu_ops", 582 "xen_irq_ops", 583 "xen_mmu_ops", 584 NULL, 585 }; 586 const char *pv_ops; 587 struct symbol *sym; 588 int idx, nr; 589 590 if (!opts.noinstr) 591 return 0; 592 593 file->pv_ops = NULL; 594 595 sym = find_symbol_by_name(file->elf, "pv_ops"); 596 if (!sym) 597 return 0; 598 599 nr = sym->len / sizeof(unsigned long); 600 file->pv_ops = calloc(nr, sizeof(struct pv_state)); 601 if (!file->pv_ops) { 602 ERROR_GLIBC("calloc"); 603 return -1; 604 } 605 606 for (idx = 0; idx < nr; idx++) 607 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 608 609 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) { 610 if (add_pv_ops(file, pv_ops)) 611 return -1; 612 } 613 614 return 0; 615 } 616 617 static bool is_livepatch_module(struct objtool_file *file) 618 { 619 struct section *sec; 620 621 if (!opts.module) 622 return false; 623 624 sec = find_section_by_name(file->elf, ".modinfo"); 625 if (!sec) 626 return false; 627 628 return memmem(sec->data->d_buf, sec_size(sec), "\0livepatch=Y", 12); 629 } 630 631 static int create_static_call_sections(struct objtool_file *file) 632 { 633 struct static_call_site *site; 634 struct section *sec; 635 struct instruction *insn; 636 struct symbol *key_sym; 637 char *key_name, *tmp; 638 int idx; 639 640 sec = find_section_by_name(file->elf, ".static_call_sites"); 641 if (sec) { 642 /* 643 * Livepatch modules may have already extracted the static call 644 * site entries to take advantage of vmlinux static call 645 * privileges. 646 */ 647 if (!file->klp) 648 WARN("file already has .static_call_sites section, skipping"); 649 650 return 0; 651 } 652 653 if (list_empty(&file->static_call_list)) 654 return 0; 655 656 idx = 0; 657 list_for_each_entry(insn, &file->static_call_list, call_node) 658 idx++; 659 660 sec = elf_create_section_pair(file->elf, ".static_call_sites", 661 sizeof(*site), idx, idx * 2); 662 if (!sec) 663 return -1; 664 665 /* Allow modules to modify the low bits of static_call_site::key */ 666 sec->sh.sh_flags |= SHF_WRITE; 667 668 idx = 0; 669 list_for_each_entry(insn, &file->static_call_list, call_node) { 670 671 /* populate reloc for 'addr' */ 672 if (!elf_init_reloc_text_sym(file->elf, sec, 673 idx * sizeof(*site), idx * 2, 674 insn->sec, insn->offset)) 675 return -1; 676 677 /* find key symbol */ 678 key_name = strdup(insn_call_dest(insn)->name); 679 if (!key_name) { 680 ERROR_GLIBC("strdup"); 681 return -1; 682 } 683 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR, 684 STATIC_CALL_TRAMP_PREFIX_LEN)) { 685 ERROR("static_call: trampoline name malformed: %s", key_name); 686 return -1; 687 } 688 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN; 689 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN); 690 691 key_sym = find_symbol_by_name(file->elf, tmp); 692 if (!key_sym) { 693 if (!opts.module || file->klp) { 694 ERROR("static_call: can't find static_call_key symbol: %s", tmp); 695 return -1; 696 } 697 698 /* 699 * For modules(), the key might not be exported, which 700 * means the module can make static calls but isn't 701 * allowed to change them. 702 * 703 * In that case we temporarily set the key to be the 704 * trampoline address. This is fixed up in 705 * static_call_add_module(). 706 */ 707 key_sym = insn_call_dest(insn); 708 } 709 710 /* populate reloc for 'key' */ 711 if (!elf_init_reloc_data_sym(file->elf, sec, 712 idx * sizeof(*site) + 4, 713 (idx * 2) + 1, key_sym, 714 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL)) 715 return -1; 716 717 idx++; 718 } 719 720 return 0; 721 } 722 723 static int create_retpoline_sites_sections(struct objtool_file *file) 724 { 725 struct instruction *insn; 726 struct section *sec; 727 int idx; 728 729 sec = find_section_by_name(file->elf, ".retpoline_sites"); 730 if (sec) { 731 WARN("file already has .retpoline_sites, skipping"); 732 return 0; 733 } 734 735 idx = 0; 736 list_for_each_entry(insn, &file->retpoline_call_list, call_node) 737 idx++; 738 739 if (!idx) 740 return 0; 741 742 sec = elf_create_section_pair(file->elf, ".retpoline_sites", 743 sizeof(int), idx, idx); 744 if (!sec) 745 return -1; 746 747 idx = 0; 748 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 749 750 if (!elf_init_reloc_text_sym(file->elf, sec, 751 idx * sizeof(int), idx, 752 insn->sec, insn->offset)) 753 return -1; 754 755 idx++; 756 } 757 758 return 0; 759 } 760 761 static int create_return_sites_sections(struct objtool_file *file) 762 { 763 struct instruction *insn; 764 struct section *sec; 765 int idx; 766 767 sec = find_section_by_name(file->elf, ".return_sites"); 768 if (sec) { 769 WARN("file already has .return_sites, skipping"); 770 return 0; 771 } 772 773 idx = 0; 774 list_for_each_entry(insn, &file->return_thunk_list, call_node) 775 idx++; 776 777 if (!idx) 778 return 0; 779 780 sec = elf_create_section_pair(file->elf, ".return_sites", 781 sizeof(int), idx, idx); 782 if (!sec) 783 return -1; 784 785 idx = 0; 786 list_for_each_entry(insn, &file->return_thunk_list, call_node) { 787 788 if (!elf_init_reloc_text_sym(file->elf, sec, 789 idx * sizeof(int), idx, 790 insn->sec, insn->offset)) 791 return -1; 792 793 idx++; 794 } 795 796 return 0; 797 } 798 799 static int create_ibt_endbr_seal_sections(struct objtool_file *file) 800 { 801 struct instruction *insn; 802 struct section *sec; 803 int idx; 804 805 sec = find_section_by_name(file->elf, ".ibt_endbr_seal"); 806 if (sec) { 807 WARN("file already has .ibt_endbr_seal, skipping"); 808 return 0; 809 } 810 811 idx = 0; 812 list_for_each_entry(insn, &file->endbr_list, call_node) 813 idx++; 814 815 if (opts.stats) { 816 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); 817 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); 818 printf("ibt: superfluous ENDBR: %d\n", idx); 819 } 820 821 if (!idx) 822 return 0; 823 824 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal", 825 sizeof(int), idx, idx); 826 if (!sec) 827 return -1; 828 829 idx = 0; 830 list_for_each_entry(insn, &file->endbr_list, call_node) { 831 832 int *site = (int *)sec->data->d_buf + idx; 833 struct symbol *sym = insn->sym; 834 *site = 0; 835 836 if (opts.module && sym && is_func_sym(sym) && 837 insn->offset == sym->offset && 838 (!strcmp(sym->name, "init_module") || 839 !strcmp(sym->name, "cleanup_module"))) { 840 ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead", 841 sym->name); 842 return -1; 843 } 844 845 if (!elf_init_reloc_text_sym(file->elf, sec, 846 idx * sizeof(int), idx, 847 insn->sec, insn->offset)) 848 return -1; 849 850 idx++; 851 } 852 853 return 0; 854 } 855 856 static int create_cfi_sections(struct objtool_file *file) 857 { 858 struct section *sec; 859 struct symbol *sym; 860 int idx; 861 862 sec = find_section_by_name(file->elf, ".cfi_sites"); 863 if (sec) { 864 WARN("file already has .cfi_sites section, skipping"); 865 return 0; 866 } 867 868 idx = 0; 869 for_each_sym(file->elf, sym) { 870 if (!is_func_sym(sym)) 871 continue; 872 873 if (strncmp(sym->name, "__cfi_", 6)) 874 continue; 875 876 idx++; 877 } 878 879 sec = elf_create_section_pair(file->elf, ".cfi_sites", 880 sizeof(unsigned int), idx, idx); 881 if (!sec) 882 return -1; 883 884 idx = 0; 885 for_each_sym(file->elf, sym) { 886 if (!is_func_sym(sym)) 887 continue; 888 889 if (strncmp(sym->name, "__cfi_", 6)) 890 continue; 891 892 if (!elf_init_reloc_text_sym(file->elf, sec, 893 idx * sizeof(unsigned int), idx, 894 sym->sec, sym->offset)) 895 return -1; 896 897 idx++; 898 } 899 900 return 0; 901 } 902 903 static int create_mcount_loc_sections(struct objtool_file *file) 904 { 905 size_t addr_size = elf_addr_size(file->elf); 906 struct instruction *insn; 907 struct section *sec; 908 int idx; 909 910 sec = find_section_by_name(file->elf, "__mcount_loc"); 911 if (sec) { 912 /* 913 * Livepatch modules have already extracted their __mcount_loc 914 * entries to cover the !CONFIG_FTRACE_MCOUNT_USE_OBJTOOL case. 915 */ 916 if (!file->klp) 917 WARN("file already has __mcount_loc section, skipping"); 918 919 return 0; 920 } 921 922 if (list_empty(&file->mcount_loc_list)) 923 return 0; 924 925 idx = 0; 926 list_for_each_entry(insn, &file->mcount_loc_list, call_node) 927 idx++; 928 929 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size, 930 idx, idx); 931 if (!sec) 932 return -1; 933 934 sec->sh.sh_addralign = addr_size; 935 936 idx = 0; 937 list_for_each_entry(insn, &file->mcount_loc_list, call_node) { 938 939 struct reloc *reloc; 940 941 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx, 942 insn->sec, insn->offset); 943 if (!reloc) 944 return -1; 945 946 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32); 947 948 idx++; 949 } 950 951 return 0; 952 } 953 954 static int create_direct_call_sections(struct objtool_file *file) 955 { 956 struct instruction *insn; 957 struct section *sec; 958 int idx; 959 960 sec = find_section_by_name(file->elf, ".call_sites"); 961 if (sec) { 962 WARN("file already has .call_sites section, skipping"); 963 return 0; 964 } 965 966 if (list_empty(&file->call_list)) 967 return 0; 968 969 idx = 0; 970 list_for_each_entry(insn, &file->call_list, call_node) 971 idx++; 972 973 sec = elf_create_section_pair(file->elf, ".call_sites", 974 sizeof(unsigned int), idx, idx); 975 if (!sec) 976 return -1; 977 978 idx = 0; 979 list_for_each_entry(insn, &file->call_list, call_node) { 980 981 if (!elf_init_reloc_text_sym(file->elf, sec, 982 idx * sizeof(unsigned int), idx, 983 insn->sec, insn->offset)) 984 return -1; 985 986 idx++; 987 } 988 989 return 0; 990 } 991 992 #ifdef BUILD_KLP 993 static int create_sym_checksum_section(struct objtool_file *file) 994 { 995 struct section *sec; 996 struct symbol *sym; 997 unsigned int idx = 0; 998 struct sym_checksum *checksum; 999 size_t entsize = sizeof(struct sym_checksum); 1000 1001 sec = find_section_by_name(file->elf, ".discard.sym_checksum"); 1002 if (sec) { 1003 if (!opts.dryrun) 1004 WARN("file already has .discard.sym_checksum section, skipping"); 1005 1006 return 0; 1007 } 1008 1009 for_each_sym(file->elf, sym) 1010 if (sym->csum.checksum) 1011 idx++; 1012 1013 if (!idx) 1014 return 0; 1015 1016 sec = elf_create_section_pair(file->elf, ".discard.sym_checksum", entsize, 1017 idx, idx); 1018 if (!sec) 1019 return -1; 1020 1021 idx = 0; 1022 for_each_sym(file->elf, sym) { 1023 if (!sym->csum.checksum) 1024 continue; 1025 1026 if (!elf_init_reloc(file->elf, sec->rsec, idx, idx * entsize, 1027 sym, 0, R_TEXT64)) 1028 return -1; 1029 1030 checksum = (struct sym_checksum *)sec->data->d_buf + idx; 1031 checksum->addr = 0; /* reloc */ 1032 checksum->checksum = sym->csum.checksum; 1033 1034 mark_sec_changed(file->elf, sec, true); 1035 1036 idx++; 1037 } 1038 1039 return 0; 1040 } 1041 #else 1042 static int create_sym_checksum_section(struct objtool_file *file) { return -EINVAL; } 1043 #endif 1044 1045 /* 1046 * Warnings shouldn't be reported for ignored functions. 1047 */ 1048 static int add_ignores(struct objtool_file *file) 1049 { 1050 struct section *rsec; 1051 struct symbol *func; 1052 struct reloc *reloc; 1053 1054 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard"); 1055 if (!rsec) 1056 return 0; 1057 1058 for_each_reloc(rsec, reloc) { 1059 switch (reloc->sym->type) { 1060 case STT_FUNC: 1061 func = reloc->sym; 1062 break; 1063 1064 case STT_SECTION: 1065 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc)); 1066 if (!func) 1067 continue; 1068 break; 1069 1070 default: 1071 ERROR("unexpected relocation symbol type in %s: %d", 1072 rsec->name, reloc->sym->type); 1073 return -1; 1074 } 1075 1076 func->ignore = true; 1077 if (func->cfunc) 1078 func->cfunc->ignore = true; 1079 } 1080 1081 return 0; 1082 } 1083 1084 /* 1085 * This is a whitelist of functions that is allowed to be called with AC set. 1086 * The list is meant to be minimal and only contains compiler instrumentation 1087 * ABI and a few functions used to implement *_{to,from}_user() functions. 1088 * 1089 * These functions must not directly change AC, but may PUSHF/POPF. 1090 */ 1091 static const char *uaccess_safe_builtin[] = { 1092 /* KASAN */ 1093 "kasan_report", 1094 "kasan_check_range", 1095 /* KASAN out-of-line */ 1096 "__asan_loadN_noabort", 1097 "__asan_load1_noabort", 1098 "__asan_load2_noabort", 1099 "__asan_load4_noabort", 1100 "__asan_load8_noabort", 1101 "__asan_load16_noabort", 1102 "__asan_storeN_noabort", 1103 "__asan_store1_noabort", 1104 "__asan_store2_noabort", 1105 "__asan_store4_noabort", 1106 "__asan_store8_noabort", 1107 "__asan_store16_noabort", 1108 "__kasan_check_read", 1109 "__kasan_check_write", 1110 /* KASAN in-line */ 1111 "__asan_report_load_n_noabort", 1112 "__asan_report_load1_noabort", 1113 "__asan_report_load2_noabort", 1114 "__asan_report_load4_noabort", 1115 "__asan_report_load8_noabort", 1116 "__asan_report_load16_noabort", 1117 "__asan_report_store_n_noabort", 1118 "__asan_report_store1_noabort", 1119 "__asan_report_store2_noabort", 1120 "__asan_report_store4_noabort", 1121 "__asan_report_store8_noabort", 1122 "__asan_report_store16_noabort", 1123 /* KCSAN */ 1124 "__kcsan_check_access", 1125 "__kcsan_mb", 1126 "__kcsan_wmb", 1127 "__kcsan_rmb", 1128 "__kcsan_release", 1129 "kcsan_found_watchpoint", 1130 "kcsan_setup_watchpoint", 1131 "kcsan_check_scoped_accesses", 1132 "kcsan_disable_current", 1133 "kcsan_enable_current_nowarn", 1134 /* KCSAN/TSAN */ 1135 "__tsan_func_entry", 1136 "__tsan_func_exit", 1137 "__tsan_read_range", 1138 "__tsan_write_range", 1139 "__tsan_read1", 1140 "__tsan_read2", 1141 "__tsan_read4", 1142 "__tsan_read8", 1143 "__tsan_read16", 1144 "__tsan_write1", 1145 "__tsan_write2", 1146 "__tsan_write4", 1147 "__tsan_write8", 1148 "__tsan_write16", 1149 "__tsan_read_write1", 1150 "__tsan_read_write2", 1151 "__tsan_read_write4", 1152 "__tsan_read_write8", 1153 "__tsan_read_write16", 1154 "__tsan_volatile_read1", 1155 "__tsan_volatile_read2", 1156 "__tsan_volatile_read4", 1157 "__tsan_volatile_read8", 1158 "__tsan_volatile_read16", 1159 "__tsan_volatile_write1", 1160 "__tsan_volatile_write2", 1161 "__tsan_volatile_write4", 1162 "__tsan_volatile_write8", 1163 "__tsan_volatile_write16", 1164 "__tsan_atomic8_load", 1165 "__tsan_atomic16_load", 1166 "__tsan_atomic32_load", 1167 "__tsan_atomic64_load", 1168 "__tsan_atomic8_store", 1169 "__tsan_atomic16_store", 1170 "__tsan_atomic32_store", 1171 "__tsan_atomic64_store", 1172 "__tsan_atomic8_exchange", 1173 "__tsan_atomic16_exchange", 1174 "__tsan_atomic32_exchange", 1175 "__tsan_atomic64_exchange", 1176 "__tsan_atomic8_fetch_add", 1177 "__tsan_atomic16_fetch_add", 1178 "__tsan_atomic32_fetch_add", 1179 "__tsan_atomic64_fetch_add", 1180 "__tsan_atomic8_fetch_sub", 1181 "__tsan_atomic16_fetch_sub", 1182 "__tsan_atomic32_fetch_sub", 1183 "__tsan_atomic64_fetch_sub", 1184 "__tsan_atomic8_fetch_and", 1185 "__tsan_atomic16_fetch_and", 1186 "__tsan_atomic32_fetch_and", 1187 "__tsan_atomic64_fetch_and", 1188 "__tsan_atomic8_fetch_or", 1189 "__tsan_atomic16_fetch_or", 1190 "__tsan_atomic32_fetch_or", 1191 "__tsan_atomic64_fetch_or", 1192 "__tsan_atomic8_fetch_xor", 1193 "__tsan_atomic16_fetch_xor", 1194 "__tsan_atomic32_fetch_xor", 1195 "__tsan_atomic64_fetch_xor", 1196 "__tsan_atomic8_fetch_nand", 1197 "__tsan_atomic16_fetch_nand", 1198 "__tsan_atomic32_fetch_nand", 1199 "__tsan_atomic64_fetch_nand", 1200 "__tsan_atomic8_compare_exchange_strong", 1201 "__tsan_atomic16_compare_exchange_strong", 1202 "__tsan_atomic32_compare_exchange_strong", 1203 "__tsan_atomic64_compare_exchange_strong", 1204 "__tsan_atomic8_compare_exchange_weak", 1205 "__tsan_atomic16_compare_exchange_weak", 1206 "__tsan_atomic32_compare_exchange_weak", 1207 "__tsan_atomic64_compare_exchange_weak", 1208 "__tsan_atomic8_compare_exchange_val", 1209 "__tsan_atomic16_compare_exchange_val", 1210 "__tsan_atomic32_compare_exchange_val", 1211 "__tsan_atomic64_compare_exchange_val", 1212 "__tsan_atomic_thread_fence", 1213 "__tsan_atomic_signal_fence", 1214 "__tsan_unaligned_read16", 1215 "__tsan_unaligned_write16", 1216 /* KCOV */ 1217 "write_comp_data", 1218 "check_kcov_mode", 1219 "__sanitizer_cov_trace_pc", 1220 "__sanitizer_cov_trace_const_cmp1", 1221 "__sanitizer_cov_trace_const_cmp2", 1222 "__sanitizer_cov_trace_const_cmp4", 1223 "__sanitizer_cov_trace_const_cmp8", 1224 "__sanitizer_cov_trace_cmp1", 1225 "__sanitizer_cov_trace_cmp2", 1226 "__sanitizer_cov_trace_cmp4", 1227 "__sanitizer_cov_trace_cmp8", 1228 "__sanitizer_cov_trace_switch", 1229 /* KMSAN */ 1230 "kmsan_copy_to_user", 1231 "kmsan_disable_current", 1232 "kmsan_enable_current", 1233 "kmsan_report", 1234 "kmsan_unpoison_entry_regs", 1235 "kmsan_unpoison_memory", 1236 "__msan_chain_origin", 1237 "__msan_get_context_state", 1238 "__msan_instrument_asm_store", 1239 "__msan_metadata_ptr_for_load_1", 1240 "__msan_metadata_ptr_for_load_2", 1241 "__msan_metadata_ptr_for_load_4", 1242 "__msan_metadata_ptr_for_load_8", 1243 "__msan_metadata_ptr_for_load_n", 1244 "__msan_metadata_ptr_for_store_1", 1245 "__msan_metadata_ptr_for_store_2", 1246 "__msan_metadata_ptr_for_store_4", 1247 "__msan_metadata_ptr_for_store_8", 1248 "__msan_metadata_ptr_for_store_n", 1249 "__msan_poison_alloca", 1250 "__msan_warning", 1251 /* UBSAN */ 1252 "ubsan_type_mismatch_common", 1253 "__ubsan_handle_type_mismatch", 1254 "__ubsan_handle_type_mismatch_v1", 1255 "__ubsan_handle_shift_out_of_bounds", 1256 "__ubsan_handle_load_invalid_value", 1257 /* KSTACK_ERASE */ 1258 "__sanitizer_cov_stack_depth", 1259 /* TRACE_BRANCH_PROFILING */ 1260 "ftrace_likely_update", 1261 /* STACKPROTECTOR */ 1262 "__stack_chk_fail", 1263 /* misc */ 1264 "csum_partial_copy_generic", 1265 "copy_mc_fragile", 1266 "copy_mc_fragile_handle_tail", 1267 "copy_mc_enhanced_fast_string", 1268 "rep_stos_alternative", 1269 "rep_movs_alternative", 1270 "__copy_user_nocache", 1271 NULL 1272 }; 1273 1274 static void add_uaccess_safe(struct objtool_file *file) 1275 { 1276 struct symbol *func; 1277 const char **name; 1278 1279 if (!opts.uaccess) 1280 return; 1281 1282 for (name = uaccess_safe_builtin; *name; name++) { 1283 func = find_symbol_by_name(file->elf, *name); 1284 if (!func) 1285 continue; 1286 1287 func->uaccess_safe = true; 1288 } 1289 } 1290 1291 /* 1292 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol 1293 * will be added to the .retpoline_sites section. 1294 */ 1295 __weak bool arch_is_retpoline(struct symbol *sym) 1296 { 1297 return false; 1298 } 1299 1300 /* 1301 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol 1302 * will be added to the .return_sites section. 1303 */ 1304 __weak bool arch_is_rethunk(struct symbol *sym) 1305 { 1306 return false; 1307 } 1308 1309 /* 1310 * Symbols that are embedded inside other instructions, because sometimes crazy 1311 * code exists. These are mostly ignored for validation purposes. 1312 */ 1313 __weak bool arch_is_embedded_insn(struct symbol *sym) 1314 { 1315 return false; 1316 } 1317 1318 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) 1319 { 1320 struct reloc *reloc; 1321 1322 if (insn->no_reloc) 1323 return NULL; 1324 1325 if (!file) 1326 return NULL; 1327 1328 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 1329 insn->offset, insn->len); 1330 if (!reloc) { 1331 insn->no_reloc = 1; 1332 return NULL; 1333 } 1334 1335 return reloc; 1336 } 1337 1338 static void remove_insn_ops(struct instruction *insn) 1339 { 1340 struct stack_op *op, *next; 1341 1342 for (op = insn->stack_ops; op; op = next) { 1343 next = op->next; 1344 free(op); 1345 } 1346 insn->stack_ops = NULL; 1347 } 1348 1349 static int annotate_call_site(struct objtool_file *file, 1350 struct instruction *insn, bool sibling) 1351 { 1352 struct reloc *reloc = insn_reloc(file, insn); 1353 struct symbol *sym = insn_call_dest(insn); 1354 1355 if (!sym) 1356 sym = reloc->sym; 1357 1358 if (sym->static_call_tramp) { 1359 list_add_tail(&insn->call_node, &file->static_call_list); 1360 return 0; 1361 } 1362 1363 if (sym->retpoline_thunk) { 1364 list_add_tail(&insn->call_node, &file->retpoline_call_list); 1365 return 0; 1366 } 1367 1368 /* 1369 * Many compilers cannot disable KCOV or sanitizer calls with a function 1370 * attribute so they need a little help, NOP out any such calls from 1371 * noinstr text. 1372 */ 1373 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) { 1374 if (reloc) 1375 set_reloc_type(file->elf, reloc, R_NONE); 1376 1377 if (elf_write_insn(file->elf, insn->sec, 1378 insn->offset, insn->len, 1379 sibling ? arch_ret_insn(insn->len) 1380 : arch_nop_insn(insn->len))) { 1381 return -1; 1382 } 1383 1384 insn->type = sibling ? INSN_RETURN : INSN_NOP; 1385 1386 if (sibling) { 1387 /* 1388 * We've replaced the tail-call JMP insn by two new 1389 * insn: RET; INT3, except we only have a single struct 1390 * insn here. Mark it retpoline_safe to avoid the SLS 1391 * warning, instead of adding another insn. 1392 */ 1393 insn->retpoline_safe = true; 1394 } 1395 1396 return 0; 1397 } 1398 1399 if (opts.mcount && sym->fentry) { 1400 if (sibling) 1401 WARN_INSN(insn, "tail call to __fentry__ !?!?"); 1402 if (opts.mnop) { 1403 if (reloc) 1404 set_reloc_type(file->elf, reloc, R_NONE); 1405 1406 if (elf_write_insn(file->elf, insn->sec, 1407 insn->offset, insn->len, 1408 arch_nop_insn(insn->len))) { 1409 return -1; 1410 } 1411 1412 insn->type = INSN_NOP; 1413 } 1414 1415 list_add_tail(&insn->call_node, &file->mcount_loc_list); 1416 return 0; 1417 } 1418 1419 if (insn->type == INSN_CALL && !insn->sec->init && 1420 !insn->_call_dest->embedded_insn) 1421 list_add_tail(&insn->call_node, &file->call_list); 1422 1423 if (!sibling && dead_end_function(file, sym)) 1424 insn->dead_end = true; 1425 1426 return 0; 1427 } 1428 1429 static int add_call_dest(struct objtool_file *file, struct instruction *insn, 1430 struct symbol *dest, bool sibling) 1431 { 1432 insn->_call_dest = dest; 1433 if (!dest) 1434 return 0; 1435 1436 /* 1437 * Whatever stack impact regular CALLs have, should be undone 1438 * by the RETURN of the called function. 1439 * 1440 * Annotated intra-function calls retain the stack_ops but 1441 * are converted to JUMP, see read_intra_function_calls(). 1442 */ 1443 remove_insn_ops(insn); 1444 1445 return annotate_call_site(file, insn, sibling); 1446 } 1447 1448 static int add_retpoline_call(struct objtool_file *file, struct instruction *insn) 1449 { 1450 /* 1451 * Retpoline calls/jumps are really dynamic calls/jumps in disguise, 1452 * so convert them accordingly. 1453 */ 1454 switch (insn->type) { 1455 case INSN_CALL: 1456 insn->type = INSN_CALL_DYNAMIC; 1457 break; 1458 case INSN_JUMP_UNCONDITIONAL: 1459 insn->type = INSN_JUMP_DYNAMIC; 1460 break; 1461 case INSN_JUMP_CONDITIONAL: 1462 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL; 1463 break; 1464 default: 1465 return 0; 1466 } 1467 1468 insn->retpoline_safe = true; 1469 1470 /* 1471 * Whatever stack impact regular CALLs have, should be undone 1472 * by the RETURN of the called function. 1473 * 1474 * Annotated intra-function calls retain the stack_ops but 1475 * are converted to JUMP, see read_intra_function_calls(). 1476 */ 1477 remove_insn_ops(insn); 1478 1479 return annotate_call_site(file, insn, false); 1480 } 1481 1482 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) 1483 { 1484 /* 1485 * Return thunk tail calls are really just returns in disguise, 1486 * so convert them accordingly. 1487 */ 1488 insn->type = INSN_RETURN; 1489 insn->retpoline_safe = true; 1490 1491 if (add) 1492 list_add_tail(&insn->call_node, &file->return_thunk_list); 1493 } 1494 1495 static bool is_first_func_insn(struct objtool_file *file, 1496 struct instruction *insn) 1497 { 1498 struct symbol *func = insn_func(insn); 1499 1500 if (!func) 1501 return false; 1502 1503 if (insn->offset == func->offset) 1504 return true; 1505 1506 /* Allow direct CALL/JMP past ENDBR */ 1507 if (opts.ibt) { 1508 struct instruction *prev = prev_insn_same_sym(file, insn); 1509 1510 if (prev && prev->type == INSN_ENDBR && 1511 insn->offset == func->offset + prev->len) 1512 return true; 1513 } 1514 1515 return false; 1516 } 1517 1518 /* 1519 * Find the destination instructions for all jumps. 1520 */ 1521 static int add_jump_destinations(struct objtool_file *file) 1522 { 1523 struct instruction *insn; 1524 struct reloc *reloc; 1525 1526 for_each_insn(file, insn) { 1527 struct symbol *func = insn_func(insn); 1528 struct instruction *dest_insn; 1529 struct section *dest_sec; 1530 struct symbol *dest_sym; 1531 unsigned long dest_off; 1532 1533 if (!is_static_jump(insn)) 1534 continue; 1535 1536 if (insn->jump_dest) { 1537 /* 1538 * handle_group_alt() may have previously set 1539 * 'jump_dest' for some alternatives. 1540 */ 1541 continue; 1542 } 1543 1544 reloc = insn_reloc(file, insn); 1545 if (!reloc) { 1546 dest_sec = insn->sec; 1547 dest_off = arch_jump_destination(insn); 1548 dest_sym = dest_sec->sym; 1549 } else { 1550 dest_sym = reloc->sym; 1551 if (is_undef_sym(dest_sym)) { 1552 if (dest_sym->retpoline_thunk) { 1553 if (add_retpoline_call(file, insn)) 1554 return -1; 1555 continue; 1556 } 1557 1558 if (dest_sym->return_thunk) { 1559 add_return_call(file, insn, true); 1560 continue; 1561 } 1562 1563 /* External symbol */ 1564 if (func) { 1565 /* External sibling call */ 1566 if (add_call_dest(file, insn, dest_sym, true)) 1567 return -1; 1568 continue; 1569 } 1570 1571 /* Non-func asm code jumping to external symbol */ 1572 continue; 1573 } 1574 1575 dest_sec = dest_sym->sec; 1576 dest_off = dest_sym->offset + arch_insn_adjusted_addend(insn, reloc); 1577 } 1578 1579 dest_insn = find_insn(file, dest_sec, dest_off); 1580 if (!dest_insn) { 1581 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); 1582 1583 /* 1584 * retbleed_untrain_ret() jumps to 1585 * __x86_return_thunk(), but objtool can't find 1586 * the thunk's starting RET instruction, 1587 * because the RET is also in the middle of 1588 * another instruction. Objtool only knows 1589 * about the outer instruction. 1590 */ 1591 if (sym && sym->embedded_insn) { 1592 add_return_call(file, insn, false); 1593 continue; 1594 } 1595 1596 /* 1597 * GCOV/KCOV dead code can jump to the end of 1598 * the function/section. 1599 */ 1600 if (file->ignore_unreachables && func && 1601 dest_sec == insn->sec && 1602 dest_off == func->offset + func->len) 1603 continue; 1604 1605 ERROR_INSN(insn, "can't find jump dest instruction at %s", 1606 offstr(dest_sec, dest_off)); 1607 return -1; 1608 } 1609 1610 if (!dest_sym || is_sec_sym(dest_sym)) { 1611 dest_sym = dest_insn->sym; 1612 if (!dest_sym) 1613 goto set_jump_dest; 1614 } 1615 1616 if (dest_sym->retpoline_thunk && dest_insn->offset == dest_sym->offset) { 1617 if (add_retpoline_call(file, insn)) 1618 return -1; 1619 continue; 1620 } 1621 1622 if (dest_sym->return_thunk && dest_insn->offset == dest_sym->offset) { 1623 add_return_call(file, insn, true); 1624 continue; 1625 } 1626 1627 if (!insn->sym || insn->sym->pfunc == dest_sym->pfunc) 1628 goto set_jump_dest; 1629 1630 /* 1631 * Internal cross-function jump. 1632 */ 1633 1634 if (is_first_func_insn(file, dest_insn)) { 1635 /* Internal sibling call */ 1636 if (add_call_dest(file, insn, dest_sym, true)) 1637 return -1; 1638 continue; 1639 } 1640 1641 set_jump_dest: 1642 insn->jump_dest = dest_insn; 1643 } 1644 1645 return 0; 1646 } 1647 1648 static struct symbol *find_call_destination(struct section *sec, unsigned long offset) 1649 { 1650 struct symbol *call_dest; 1651 1652 call_dest = find_func_by_offset(sec, offset); 1653 if (!call_dest) 1654 call_dest = find_symbol_by_offset(sec, offset); 1655 1656 return call_dest; 1657 } 1658 1659 /* 1660 * Find the destination instructions for all calls. 1661 */ 1662 static int add_call_destinations(struct objtool_file *file) 1663 { 1664 struct instruction *insn; 1665 unsigned long dest_off; 1666 struct symbol *dest; 1667 struct reloc *reloc; 1668 1669 for_each_insn(file, insn) { 1670 struct symbol *func = insn_func(insn); 1671 if (insn->type != INSN_CALL) 1672 continue; 1673 1674 reloc = insn_reloc(file, insn); 1675 if (!reloc) { 1676 dest_off = arch_jump_destination(insn); 1677 dest = find_call_destination(insn->sec, dest_off); 1678 1679 if (add_call_dest(file, insn, dest, false)) 1680 return -1; 1681 1682 if (func && func->ignore) 1683 continue; 1684 1685 if (!insn_call_dest(insn)) { 1686 ERROR_INSN(insn, "unannotated intra-function call"); 1687 return -1; 1688 } 1689 1690 if (func && !is_func_sym(insn_call_dest(insn))) { 1691 ERROR_INSN(insn, "unsupported call to non-function"); 1692 return -1; 1693 } 1694 1695 } else if (is_sec_sym(reloc->sym)) { 1696 dest_off = arch_insn_adjusted_addend(insn, reloc); 1697 dest = find_call_destination(reloc->sym->sec, dest_off); 1698 if (!dest) { 1699 ERROR_INSN(insn, "can't find call dest symbol at %s+0x%lx", 1700 reloc->sym->sec->name, dest_off); 1701 return -1; 1702 } 1703 1704 if (add_call_dest(file, insn, dest, false)) 1705 return -1; 1706 1707 } else if (reloc->sym->retpoline_thunk) { 1708 if (add_retpoline_call(file, insn)) 1709 return -1; 1710 1711 } else { 1712 if (add_call_dest(file, insn, reloc->sym, false)) 1713 return -1; 1714 } 1715 } 1716 1717 return 0; 1718 } 1719 1720 /* 1721 * The .alternatives section requires some extra special care over and above 1722 * other special sections because alternatives are patched in place. 1723 */ 1724 static int handle_group_alt(struct objtool_file *file, 1725 struct special_alt *special_alt, 1726 struct instruction *orig_insn, 1727 struct instruction **new_insn) 1728 { 1729 struct instruction *last_new_insn = NULL, *insn, *nop = NULL; 1730 struct alt_group *orig_alt_group, *new_alt_group; 1731 unsigned long dest_off; 1732 1733 orig_alt_group = orig_insn->alt_group; 1734 if (!orig_alt_group) { 1735 struct instruction *last_orig_insn = NULL; 1736 1737 orig_alt_group = calloc(1, sizeof(*orig_alt_group)); 1738 if (!orig_alt_group) { 1739 ERROR_GLIBC("calloc"); 1740 return -1; 1741 } 1742 orig_alt_group->cfi = calloc(special_alt->orig_len, 1743 sizeof(struct cfi_state *)); 1744 if (!orig_alt_group->cfi) { 1745 ERROR_GLIBC("calloc"); 1746 return -1; 1747 } 1748 1749 insn = orig_insn; 1750 sec_for_each_insn_from(file, insn) { 1751 if (insn->offset >= special_alt->orig_off + special_alt->orig_len) 1752 break; 1753 1754 insn->alt_group = orig_alt_group; 1755 last_orig_insn = insn; 1756 } 1757 orig_alt_group->orig_group = NULL; 1758 orig_alt_group->first_insn = orig_insn; 1759 orig_alt_group->last_insn = last_orig_insn; 1760 orig_alt_group->nop = NULL; 1761 orig_alt_group->ignore = orig_insn->ignore_alts; 1762 } else { 1763 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - 1764 orig_alt_group->first_insn->offset != special_alt->orig_len) { 1765 ERROR_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d", 1766 orig_alt_group->last_insn->offset + 1767 orig_alt_group->last_insn->len - 1768 orig_alt_group->first_insn->offset, 1769 special_alt->orig_len); 1770 return -1; 1771 } 1772 } 1773 1774 new_alt_group = calloc(1, sizeof(*new_alt_group)); 1775 if (!new_alt_group) { 1776 ERROR_GLIBC("calloc"); 1777 return -1; 1778 } 1779 1780 if (special_alt->new_len < special_alt->orig_len) { 1781 /* 1782 * Insert a fake nop at the end to make the replacement 1783 * alt_group the same size as the original. This is needed to 1784 * allow propagate_alt_cfi() to do its magic. When the last 1785 * instruction affects the stack, the instruction after it (the 1786 * nop) will propagate the new state to the shared CFI array. 1787 */ 1788 nop = calloc(1, sizeof(*nop)); 1789 if (!nop) { 1790 ERROR_GLIBC("calloc"); 1791 return -1; 1792 } 1793 memset(nop, 0, sizeof(*nop)); 1794 1795 nop->sec = special_alt->new_sec; 1796 nop->offset = special_alt->new_off + special_alt->new_len; 1797 nop->len = special_alt->orig_len - special_alt->new_len; 1798 nop->type = INSN_NOP; 1799 nop->sym = orig_insn->sym; 1800 nop->alt_group = new_alt_group; 1801 nop->fake = 1; 1802 } 1803 1804 if (!special_alt->new_len) { 1805 *new_insn = nop; 1806 goto end; 1807 } 1808 1809 insn = *new_insn; 1810 sec_for_each_insn_from(file, insn) { 1811 struct reloc *alt_reloc; 1812 1813 if (insn->offset >= special_alt->new_off + special_alt->new_len) 1814 break; 1815 1816 last_new_insn = insn; 1817 1818 insn->sym = orig_insn->sym; 1819 insn->alt_group = new_alt_group; 1820 1821 /* 1822 * Since alternative replacement code is copy/pasted by the 1823 * kernel after applying relocations, generally such code can't 1824 * have relative-address relocation references to outside the 1825 * .altinstr_replacement section, unless the arch's 1826 * alternatives code can adjust the relative offsets 1827 * accordingly. 1828 */ 1829 alt_reloc = insn_reloc(file, insn); 1830 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) && 1831 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { 1832 1833 ERROR_INSN(insn, "unsupported relocation in alternatives section"); 1834 return -1; 1835 } 1836 1837 if (!is_static_jump(insn)) 1838 continue; 1839 1840 if (!insn->immediate) 1841 continue; 1842 1843 dest_off = arch_jump_destination(insn); 1844 if (dest_off == special_alt->new_off + special_alt->new_len) { 1845 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn); 1846 if (!insn->jump_dest) { 1847 ERROR_INSN(insn, "can't find alternative jump destination"); 1848 return -1; 1849 } 1850 } 1851 } 1852 1853 if (!last_new_insn) { 1854 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1855 "can't find last new alternative instruction"); 1856 return -1; 1857 } 1858 1859 end: 1860 new_alt_group->orig_group = orig_alt_group; 1861 new_alt_group->first_insn = *new_insn; 1862 new_alt_group->last_insn = last_new_insn; 1863 new_alt_group->nop = nop; 1864 new_alt_group->ignore = (*new_insn)->ignore_alts; 1865 new_alt_group->cfi = orig_alt_group->cfi; 1866 return 0; 1867 } 1868 1869 /* 1870 * A jump table entry can either convert a nop to a jump or a jump to a nop. 1871 * If the original instruction is a jump, make the alt entry an effective nop 1872 * by just skipping the original instruction. 1873 */ 1874 static int handle_jump_alt(struct objtool_file *file, 1875 struct special_alt *special_alt, 1876 struct instruction *orig_insn, 1877 struct instruction **new_insn) 1878 { 1879 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && 1880 orig_insn->type != INSN_NOP) { 1881 1882 ERROR_INSN(orig_insn, "unsupported instruction at jump label"); 1883 return -1; 1884 } 1885 1886 if (opts.hack_jump_label && special_alt->key_addend & 2) { 1887 struct reloc *reloc = insn_reloc(file, orig_insn); 1888 1889 if (reloc) 1890 set_reloc_type(file->elf, reloc, R_NONE); 1891 1892 if (elf_write_insn(file->elf, orig_insn->sec, 1893 orig_insn->offset, orig_insn->len, 1894 arch_nop_insn(orig_insn->len))) { 1895 return -1; 1896 } 1897 1898 orig_insn->type = INSN_NOP; 1899 } 1900 1901 if (orig_insn->type == INSN_NOP) { 1902 if (orig_insn->len == 2) 1903 file->jl_nop_short++; 1904 else 1905 file->jl_nop_long++; 1906 1907 return 0; 1908 } 1909 1910 if (orig_insn->len == 2) 1911 file->jl_short++; 1912 else 1913 file->jl_long++; 1914 1915 *new_insn = next_insn_same_sec(file, orig_insn); 1916 return 0; 1917 } 1918 1919 /* 1920 * Read all the special sections which have alternate instructions which can be 1921 * patched in or redirected to at runtime. Each instruction having alternate 1922 * instruction(s) has them added to its insn->alts list, which will be 1923 * traversed in validate_branch(). 1924 */ 1925 static int add_special_section_alts(struct objtool_file *file) 1926 { 1927 struct list_head special_alts; 1928 struct instruction *orig_insn, *new_insn; 1929 struct special_alt *special_alt, *tmp; 1930 struct alternative *alt; 1931 1932 if (special_get_alts(file->elf, &special_alts)) 1933 return -1; 1934 1935 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) { 1936 1937 orig_insn = find_insn(file, special_alt->orig_sec, 1938 special_alt->orig_off); 1939 if (!orig_insn) { 1940 ERROR_FUNC(special_alt->orig_sec, special_alt->orig_off, 1941 "special: can't find orig instruction"); 1942 return -1; 1943 } 1944 1945 new_insn = NULL; 1946 if (!special_alt->group || special_alt->new_len) { 1947 new_insn = find_insn(file, special_alt->new_sec, 1948 special_alt->new_off); 1949 if (!new_insn) { 1950 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1951 "special: can't find new instruction"); 1952 return -1; 1953 } 1954 } 1955 1956 if (special_alt->group) { 1957 if (!special_alt->orig_len) { 1958 ERROR_INSN(orig_insn, "empty alternative entry"); 1959 continue; 1960 } 1961 1962 if (handle_group_alt(file, special_alt, orig_insn, &new_insn)) 1963 return -1; 1964 1965 } else if (special_alt->jump_or_nop) { 1966 if (handle_jump_alt(file, special_alt, orig_insn, &new_insn)) 1967 return -1; 1968 } 1969 1970 alt = calloc(1, sizeof(*alt)); 1971 if (!alt) { 1972 ERROR_GLIBC("calloc"); 1973 return -1; 1974 } 1975 1976 alt->insn = new_insn; 1977 alt->next = orig_insn->alts; 1978 orig_insn->alts = alt; 1979 1980 list_del(&special_alt->list); 1981 free(special_alt); 1982 } 1983 1984 if (opts.stats) { 1985 printf("jl\\\tNOP\tJMP\n"); 1986 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); 1987 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); 1988 } 1989 1990 return 0; 1991 } 1992 1993 __weak unsigned long arch_jump_table_sym_offset(struct reloc *reloc, struct reloc *table) 1994 { 1995 return reloc->sym->offset + reloc_addend(reloc); 1996 } 1997 1998 static int add_jump_table(struct objtool_file *file, struct instruction *insn) 1999 { 2000 unsigned long table_size = insn_jump_table_size(insn); 2001 struct symbol *pfunc = insn_func(insn)->pfunc; 2002 struct reloc *table = insn_jump_table(insn); 2003 struct instruction *dest_insn; 2004 unsigned int prev_offset = 0; 2005 struct reloc *reloc = table; 2006 struct alternative *alt; 2007 unsigned long sym_offset; 2008 2009 /* 2010 * Each @reloc is a switch table relocation which points to the target 2011 * instruction. 2012 */ 2013 for_each_reloc_from(table->sec, reloc) { 2014 2015 /* Check for the end of the table: */ 2016 if (table_size && reloc_offset(reloc) - reloc_offset(table) >= table_size) 2017 break; 2018 if (reloc != table && is_jump_table(reloc)) 2019 break; 2020 2021 /* Make sure the table entries are consecutive: */ 2022 if (prev_offset && reloc_offset(reloc) != prev_offset + arch_reloc_size(reloc)) 2023 break; 2024 2025 sym_offset = arch_jump_table_sym_offset(reloc, table); 2026 2027 /* Detect function pointers from contiguous objects: */ 2028 if (reloc->sym->sec == pfunc->sec && sym_offset == pfunc->offset) 2029 break; 2030 2031 /* 2032 * Clang sometimes leaves dangling unused jump table entries 2033 * which point to the end of the function. Ignore them. 2034 */ 2035 if (reloc->sym->sec == pfunc->sec && 2036 sym_offset == pfunc->offset + pfunc->len) 2037 goto next; 2038 2039 dest_insn = find_insn(file, reloc->sym->sec, sym_offset); 2040 if (!dest_insn) 2041 break; 2042 2043 /* Make sure the destination is in the same function: */ 2044 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc) 2045 break; 2046 2047 alt = calloc(1, sizeof(*alt)); 2048 if (!alt) { 2049 ERROR_GLIBC("calloc"); 2050 return -1; 2051 } 2052 2053 alt->insn = dest_insn; 2054 alt->next = insn->alts; 2055 insn->alts = alt; 2056 next: 2057 prev_offset = reloc_offset(reloc); 2058 } 2059 2060 if (!prev_offset) { 2061 ERROR_INSN(insn, "can't find switch jump table"); 2062 return -1; 2063 } 2064 2065 return 0; 2066 } 2067 2068 /* 2069 * find_jump_table() - Given a dynamic jump, find the switch jump table 2070 * associated with it. 2071 */ 2072 static void find_jump_table(struct objtool_file *file, struct symbol *func, 2073 struct instruction *insn) 2074 { 2075 struct reloc *table_reloc; 2076 struct instruction *dest_insn, *orig_insn = insn; 2077 unsigned long table_size; 2078 unsigned long sym_offset; 2079 2080 /* 2081 * Backward search using the @first_jump_src links, these help avoid 2082 * much of the 'in between' code. Which avoids us getting confused by 2083 * it. 2084 */ 2085 for (; 2086 insn && insn_func(insn) && insn_func(insn)->pfunc == func; 2087 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) { 2088 2089 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC) 2090 break; 2091 2092 /* allow small jumps within the range */ 2093 if (insn->type == INSN_JUMP_UNCONDITIONAL && 2094 insn->jump_dest && 2095 (insn->jump_dest->offset <= insn->offset || 2096 insn->jump_dest->offset > orig_insn->offset)) 2097 break; 2098 2099 table_reloc = arch_find_switch_table(file, insn, &table_size); 2100 if (!table_reloc) 2101 continue; 2102 2103 sym_offset = table_reloc->sym->offset + reloc_addend(table_reloc); 2104 2105 dest_insn = find_insn(file, table_reloc->sym->sec, sym_offset); 2106 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func) 2107 continue; 2108 2109 set_jump_table(table_reloc); 2110 orig_insn->_jump_table = table_reloc; 2111 orig_insn->_jump_table_size = table_size; 2112 2113 break; 2114 } 2115 } 2116 2117 /* 2118 * First pass: Mark the head of each jump table so that in the next pass, 2119 * we know when a given jump table ends and the next one starts. 2120 */ 2121 static void mark_func_jump_tables(struct objtool_file *file, 2122 struct symbol *func) 2123 { 2124 struct instruction *insn, *last = NULL; 2125 2126 func_for_each_insn(file, func, insn) { 2127 if (!last) 2128 last = insn; 2129 2130 /* 2131 * Store back-pointers for unconditional forward jumps such 2132 * that find_jump_table() can back-track using those and 2133 * avoid some potentially confusing code. 2134 */ 2135 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && 2136 insn->offset > last->offset && 2137 insn->jump_dest->offset > insn->offset && 2138 !insn->jump_dest->first_jump_src) { 2139 2140 insn->jump_dest->first_jump_src = insn; 2141 last = insn->jump_dest; 2142 } 2143 2144 if (insn->type != INSN_JUMP_DYNAMIC) 2145 continue; 2146 2147 find_jump_table(file, func, insn); 2148 } 2149 } 2150 2151 static int add_func_jump_tables(struct objtool_file *file, 2152 struct symbol *func) 2153 { 2154 struct instruction *insn; 2155 2156 func_for_each_insn(file, func, insn) { 2157 if (!insn_jump_table(insn)) 2158 continue; 2159 2160 if (add_jump_table(file, insn)) 2161 return -1; 2162 } 2163 2164 return 0; 2165 } 2166 2167 /* 2168 * For some switch statements, gcc generates a jump table in the .rodata 2169 * section which contains a list of addresses within the function to jump to. 2170 * This finds these jump tables and adds them to the insn->alts lists. 2171 */ 2172 static int add_jump_table_alts(struct objtool_file *file) 2173 { 2174 struct symbol *func; 2175 2176 if (!file->rodata) 2177 return 0; 2178 2179 for_each_sym(file->elf, func) { 2180 if (!is_func_sym(func) || func->alias != func) 2181 continue; 2182 2183 mark_func_jump_tables(file, func); 2184 if (add_func_jump_tables(file, func)) 2185 return -1; 2186 } 2187 2188 return 0; 2189 } 2190 2191 static void set_func_state(struct cfi_state *state) 2192 { 2193 state->cfa = initial_func_cfi.cfa; 2194 memcpy(&state->regs, &initial_func_cfi.regs, 2195 CFI_NUM_REGS * sizeof(struct cfi_reg)); 2196 state->stack_size = initial_func_cfi.cfa.offset; 2197 state->type = UNWIND_HINT_TYPE_CALL; 2198 } 2199 2200 static int read_unwind_hints(struct objtool_file *file) 2201 { 2202 struct cfi_state cfi = init_cfi; 2203 struct section *sec; 2204 struct unwind_hint *hint; 2205 struct instruction *insn; 2206 struct reloc *reloc; 2207 unsigned long offset; 2208 int i; 2209 2210 sec = find_section_by_name(file->elf, ".discard.unwind_hints"); 2211 if (!sec) 2212 return 0; 2213 2214 if (!sec->rsec) { 2215 ERROR("missing .rela.discard.unwind_hints section"); 2216 return -1; 2217 } 2218 2219 if (sec_size(sec) % sizeof(struct unwind_hint)) { 2220 ERROR("struct unwind_hint size mismatch"); 2221 return -1; 2222 } 2223 2224 file->hints = true; 2225 2226 for (i = 0; i < sec_size(sec) / sizeof(struct unwind_hint); i++) { 2227 hint = (struct unwind_hint *)sec->data->d_buf + i; 2228 2229 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); 2230 if (!reloc) { 2231 ERROR("can't find reloc for unwind_hints[%d]", i); 2232 return -1; 2233 } 2234 2235 offset = reloc->sym->offset + reloc_addend(reloc); 2236 2237 insn = find_insn(file, reloc->sym->sec, offset); 2238 if (!insn) { 2239 ERROR("can't find insn for unwind_hints[%d]", i); 2240 return -1; 2241 } 2242 2243 insn->hint = true; 2244 2245 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) { 2246 insn->cfi = &force_undefined_cfi; 2247 continue; 2248 } 2249 2250 if (hint->type == UNWIND_HINT_TYPE_SAVE) { 2251 insn->hint = false; 2252 insn->save = true; 2253 continue; 2254 } 2255 2256 if (hint->type == UNWIND_HINT_TYPE_RESTORE) { 2257 insn->restore = true; 2258 continue; 2259 } 2260 2261 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 2262 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 2263 2264 if (sym && is_global_sym(sym)) { 2265 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { 2266 ERROR_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR"); 2267 return -1; 2268 } 2269 } 2270 } 2271 2272 if (hint->type == UNWIND_HINT_TYPE_FUNC) { 2273 insn->cfi = &func_cfi; 2274 continue; 2275 } 2276 2277 if (insn->cfi) 2278 cfi = *(insn->cfi); 2279 2280 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { 2281 ERROR_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg); 2282 return -1; 2283 } 2284 2285 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); 2286 cfi.type = hint->type; 2287 cfi.signal = hint->signal; 2288 2289 insn->cfi = cfi_hash_find_or_add(&cfi); 2290 } 2291 2292 return 0; 2293 } 2294 2295 static int read_annotate(struct objtool_file *file, 2296 int (*func)(struct objtool_file *file, int type, struct instruction *insn)) 2297 { 2298 struct section *sec; 2299 struct instruction *insn; 2300 struct reloc *reloc; 2301 uint64_t offset; 2302 int type; 2303 2304 sec = find_section_by_name(file->elf, ".discard.annotate_insn"); 2305 if (!sec) 2306 return 0; 2307 2308 if (!sec->rsec) 2309 return 0; 2310 2311 if (sec->sh.sh_entsize != 8) { 2312 static bool warned = false; 2313 if (!warned && opts.verbose) { 2314 WARN("%s: dodgy linker, sh_entsize != 8", sec->name); 2315 warned = true; 2316 } 2317 sec->sh.sh_entsize = 8; 2318 } 2319 2320 if (sec_num_entries(sec) != sec_num_entries(sec->rsec)) { 2321 ERROR("bad .discard.annotate_insn section: missing relocs"); 2322 return -1; 2323 } 2324 2325 for_each_reloc(sec->rsec, reloc) { 2326 type = annotype(file->elf, sec, reloc); 2327 offset = reloc->sym->offset + reloc_addend(reloc); 2328 insn = find_insn(file, reloc->sym->sec, offset); 2329 2330 if (!insn) { 2331 ERROR("bad .discard.annotate_insn entry: %d of type %d", reloc_idx(reloc), type); 2332 return -1; 2333 } 2334 2335 if (func(file, type, insn)) 2336 return -1; 2337 } 2338 2339 return 0; 2340 } 2341 2342 static int __annotate_early(struct objtool_file *file, int type, struct instruction *insn) 2343 { 2344 switch (type) { 2345 2346 /* Must be before add_special_section_alts() */ 2347 case ANNOTYPE_IGNORE_ALTS: 2348 insn->ignore_alts = true; 2349 break; 2350 2351 /* 2352 * Must be before read_unwind_hints() since that needs insn->noendbr. 2353 */ 2354 case ANNOTYPE_NOENDBR: 2355 insn->noendbr = 1; 2356 break; 2357 2358 default: 2359 break; 2360 } 2361 2362 return 0; 2363 } 2364 2365 static int __annotate_ifc(struct objtool_file *file, int type, struct instruction *insn) 2366 { 2367 unsigned long dest_off; 2368 2369 if (type != ANNOTYPE_INTRA_FUNCTION_CALL) 2370 return 0; 2371 2372 if (insn->type != INSN_CALL) { 2373 ERROR_INSN(insn, "intra_function_call not a direct call"); 2374 return -1; 2375 } 2376 2377 /* 2378 * Treat intra-function CALLs as JMPs, but with a stack_op. 2379 * See add_call_destinations(), which strips stack_ops from 2380 * normal CALLs. 2381 */ 2382 insn->type = INSN_JUMP_UNCONDITIONAL; 2383 2384 dest_off = arch_jump_destination(insn); 2385 insn->jump_dest = find_insn(file, insn->sec, dest_off); 2386 if (!insn->jump_dest) { 2387 ERROR_INSN(insn, "can't find call dest at %s+0x%lx", 2388 insn->sec->name, dest_off); 2389 return -1; 2390 } 2391 2392 return 0; 2393 } 2394 2395 static int __annotate_late(struct objtool_file *file, int type, struct instruction *insn) 2396 { 2397 struct symbol *sym; 2398 2399 switch (type) { 2400 case ANNOTYPE_NOENDBR: 2401 /* early */ 2402 break; 2403 2404 case ANNOTYPE_RETPOLINE_SAFE: 2405 if (insn->type != INSN_JUMP_DYNAMIC && 2406 insn->type != INSN_CALL_DYNAMIC && 2407 insn->type != INSN_RETURN && 2408 insn->type != INSN_NOP) { 2409 ERROR_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); 2410 return -1; 2411 } 2412 2413 insn->retpoline_safe = true; 2414 break; 2415 2416 case ANNOTYPE_INSTR_BEGIN: 2417 insn->instr++; 2418 break; 2419 2420 case ANNOTYPE_INSTR_END: 2421 insn->instr--; 2422 break; 2423 2424 case ANNOTYPE_UNRET_BEGIN: 2425 insn->unret = 1; 2426 break; 2427 2428 case ANNOTYPE_IGNORE_ALTS: 2429 /* early */ 2430 break; 2431 2432 case ANNOTYPE_INTRA_FUNCTION_CALL: 2433 /* ifc */ 2434 break; 2435 2436 case ANNOTYPE_REACHABLE: 2437 insn->dead_end = false; 2438 break; 2439 2440 case ANNOTYPE_NOCFI: 2441 sym = insn->sym; 2442 if (!sym) { 2443 ERROR_INSN(insn, "dodgy NOCFI annotation"); 2444 return -1; 2445 } 2446 insn->sym->nocfi = 1; 2447 break; 2448 2449 default: 2450 ERROR_INSN(insn, "Unknown annotation type: %d", type); 2451 return -1; 2452 } 2453 2454 return 0; 2455 } 2456 2457 /* 2458 * Return true if name matches an instrumentation function, where calls to that 2459 * function from noinstr code can safely be removed, but compilers won't do so. 2460 */ 2461 static bool is_profiling_func(const char *name) 2462 { 2463 /* 2464 * Many compilers cannot disable KCOV with a function attribute. 2465 */ 2466 if (!strncmp(name, "__sanitizer_cov_", 16)) 2467 return true; 2468 2469 return false; 2470 } 2471 2472 static int classify_symbols(struct objtool_file *file) 2473 { 2474 struct symbol *func; 2475 2476 for_each_sym(file->elf, func) { 2477 if (is_notype_sym(func) && strstarts(func->name, ".L")) 2478 func->local_label = true; 2479 2480 if (!is_global_sym(func)) 2481 continue; 2482 2483 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, 2484 strlen(STATIC_CALL_TRAMP_PREFIX_STR))) 2485 func->static_call_tramp = true; 2486 2487 if (arch_is_retpoline(func)) 2488 func->retpoline_thunk = true; 2489 2490 if (arch_is_rethunk(func)) 2491 func->return_thunk = true; 2492 2493 if (arch_is_embedded_insn(func)) 2494 func->embedded_insn = true; 2495 2496 if (arch_ftrace_match(func->name)) 2497 func->fentry = true; 2498 2499 if (is_profiling_func(func->name)) 2500 func->profiling_func = true; 2501 } 2502 2503 return 0; 2504 } 2505 2506 static void mark_rodata(struct objtool_file *file) 2507 { 2508 struct section *sec; 2509 bool found = false; 2510 2511 /* 2512 * Search for the following rodata sections, each of which can 2513 * potentially contain jump tables: 2514 * 2515 * - .rodata: can contain GCC switch tables 2516 * - .rodata.<func>: same, if -fdata-sections is being used 2517 * - .data.rel.ro.c_jump_table: contains C annotated jump tables 2518 * 2519 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2520 */ 2521 for_each_sec(file->elf, sec) { 2522 if ((!strncmp(sec->name, ".rodata", 7) && 2523 !strstr(sec->name, ".str1.")) || 2524 !strncmp(sec->name, ".data.rel.ro", 12)) { 2525 sec->rodata = true; 2526 found = true; 2527 } 2528 } 2529 2530 file->rodata = found; 2531 } 2532 2533 static void mark_holes(struct objtool_file *file) 2534 { 2535 struct instruction *insn; 2536 bool in_hole = false; 2537 2538 if (!opts.link) 2539 return; 2540 2541 /* 2542 * Whole archive runs might encounter dead code from weak symbols. 2543 * This is where the linker will have dropped the weak symbol in 2544 * favour of a regular symbol, but leaves the code in place. 2545 */ 2546 for_each_insn(file, insn) { 2547 if (insn->sym || !find_symbol_hole_containing(insn->sec, insn->offset)) { 2548 in_hole = false; 2549 continue; 2550 } 2551 2552 /* Skip function padding and pfx code */ 2553 if (!in_hole && insn->type == INSN_NOP) 2554 continue; 2555 2556 in_hole = true; 2557 insn->hole = 1; 2558 2559 /* 2560 * If this hole jumps to a .cold function, mark it ignore. 2561 */ 2562 if (insn->jump_dest) { 2563 struct symbol *dest_func = insn_func(insn->jump_dest); 2564 2565 if (dest_func && dest_func->cold) 2566 dest_func->ignore = true; 2567 } 2568 } 2569 } 2570 2571 static bool validate_branch_enabled(void) 2572 { 2573 return opts.stackval || 2574 opts.orc || 2575 opts.uaccess || 2576 opts.checksum; 2577 } 2578 2579 static int decode_sections(struct objtool_file *file) 2580 { 2581 file->klp = is_livepatch_module(file); 2582 2583 mark_rodata(file); 2584 2585 if (init_pv_ops(file)) 2586 return -1; 2587 2588 /* 2589 * Must be before add_{jump_call}_destination. 2590 */ 2591 if (classify_symbols(file)) 2592 return -1; 2593 2594 if (decode_instructions(file)) 2595 return -1; 2596 2597 if (add_ignores(file)) 2598 return -1; 2599 2600 add_uaccess_safe(file); 2601 2602 if (read_annotate(file, __annotate_early)) 2603 return -1; 2604 2605 /* 2606 * Must be before add_jump_destinations(), which depends on 'func' 2607 * being set for alternatives, to enable proper sibling call detection. 2608 */ 2609 if (validate_branch_enabled() || opts.noinstr || opts.hack_jump_label) { 2610 if (add_special_section_alts(file)) 2611 return -1; 2612 } 2613 2614 if (add_jump_destinations(file)) 2615 return -1; 2616 2617 /* 2618 * Must be before add_call_destination(); it changes INSN_CALL to 2619 * INSN_JUMP. 2620 */ 2621 if (read_annotate(file, __annotate_ifc)) 2622 return -1; 2623 2624 if (add_call_destinations(file)) 2625 return -1; 2626 2627 if (add_jump_table_alts(file)) 2628 return -1; 2629 2630 if (read_unwind_hints(file)) 2631 return -1; 2632 2633 /* Must be after add_jump_destinations() */ 2634 mark_holes(file); 2635 2636 /* 2637 * Must be after add_call_destinations() such that it can override 2638 * dead_end_function() marks. 2639 */ 2640 if (read_annotate(file, __annotate_late)) 2641 return -1; 2642 2643 return 0; 2644 } 2645 2646 static bool is_special_call(struct instruction *insn) 2647 { 2648 if (insn->type == INSN_CALL) { 2649 struct symbol *dest = insn_call_dest(insn); 2650 2651 if (!dest) 2652 return false; 2653 2654 if (dest->fentry || dest->embedded_insn) 2655 return true; 2656 } 2657 2658 return false; 2659 } 2660 2661 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state) 2662 { 2663 struct cfi_state *cfi = &state->cfi; 2664 int i; 2665 2666 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap) 2667 return true; 2668 2669 if (cfi->cfa.offset != initial_func_cfi.cfa.offset) 2670 return true; 2671 2672 if (cfi->stack_size != initial_func_cfi.cfa.offset) 2673 return true; 2674 2675 for (i = 0; i < CFI_NUM_REGS; i++) { 2676 if (cfi->regs[i].base != initial_func_cfi.regs[i].base || 2677 cfi->regs[i].offset != initial_func_cfi.regs[i].offset) 2678 return true; 2679 } 2680 2681 return false; 2682 } 2683 2684 static bool check_reg_frame_pos(const struct cfi_reg *reg, 2685 int expected_offset) 2686 { 2687 return reg->base == CFI_CFA && 2688 reg->offset == expected_offset; 2689 } 2690 2691 static bool has_valid_stack_frame(struct insn_state *state) 2692 { 2693 struct cfi_state *cfi = &state->cfi; 2694 2695 if (cfi->cfa.base == CFI_BP && 2696 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) && 2697 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8)) 2698 return true; 2699 2700 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP) 2701 return true; 2702 2703 return false; 2704 } 2705 2706 static int update_cfi_state_regs(struct instruction *insn, 2707 struct cfi_state *cfi, 2708 struct stack_op *op) 2709 { 2710 struct cfi_reg *cfa = &cfi->cfa; 2711 2712 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT) 2713 return 0; 2714 2715 /* push */ 2716 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF) 2717 cfa->offset += 8; 2718 2719 /* pop */ 2720 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF) 2721 cfa->offset -= 8; 2722 2723 /* add immediate to sp */ 2724 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD && 2725 op->dest.reg == CFI_SP && op->src.reg == CFI_SP) 2726 cfa->offset -= op->src.offset; 2727 2728 return 0; 2729 } 2730 2731 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset) 2732 { 2733 if (arch_callee_saved_reg(reg) && 2734 cfi->regs[reg].base == CFI_UNDEFINED) { 2735 cfi->regs[reg].base = base; 2736 cfi->regs[reg].offset = offset; 2737 } 2738 } 2739 2740 static void restore_reg(struct cfi_state *cfi, unsigned char reg) 2741 { 2742 cfi->regs[reg].base = initial_func_cfi.regs[reg].base; 2743 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset; 2744 } 2745 2746 /* 2747 * A note about DRAP stack alignment: 2748 * 2749 * GCC has the concept of a DRAP register, which is used to help keep track of 2750 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP 2751 * register. The typical DRAP pattern is: 2752 * 2753 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10 2754 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp 2755 * 41 ff 72 f8 pushq -0x8(%r10) 2756 * 55 push %rbp 2757 * 48 89 e5 mov %rsp,%rbp 2758 * (more pushes) 2759 * 41 52 push %r10 2760 * ... 2761 * 41 5a pop %r10 2762 * (more pops) 2763 * 5d pop %rbp 2764 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2765 * c3 retq 2766 * 2767 * There are some variations in the epilogues, like: 2768 * 2769 * 5b pop %rbx 2770 * 41 5a pop %r10 2771 * 41 5c pop %r12 2772 * 41 5d pop %r13 2773 * 41 5e pop %r14 2774 * c9 leaveq 2775 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2776 * c3 retq 2777 * 2778 * and: 2779 * 2780 * 4c 8b 55 e8 mov -0x18(%rbp),%r10 2781 * 48 8b 5d e0 mov -0x20(%rbp),%rbx 2782 * 4c 8b 65 f0 mov -0x10(%rbp),%r12 2783 * 4c 8b 6d f8 mov -0x8(%rbp),%r13 2784 * c9 leaveq 2785 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2786 * c3 retq 2787 * 2788 * Sometimes r13 is used as the DRAP register, in which case it's saved and 2789 * restored beforehand: 2790 * 2791 * 41 55 push %r13 2792 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13 2793 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp 2794 * ... 2795 * 49 8d 65 f0 lea -0x10(%r13),%rsp 2796 * 41 5d pop %r13 2797 * c3 retq 2798 */ 2799 static int update_cfi_state(struct instruction *insn, 2800 struct instruction *next_insn, 2801 struct cfi_state *cfi, struct stack_op *op) 2802 { 2803 struct cfi_reg *cfa = &cfi->cfa; 2804 struct cfi_reg *regs = cfi->regs; 2805 2806 /* ignore UNWIND_HINT_UNDEFINED regions */ 2807 if (cfi->force_undefined) 2808 return 0; 2809 2810 /* stack operations don't make sense with an undefined CFA */ 2811 if (cfa->base == CFI_UNDEFINED) { 2812 if (insn_func(insn)) { 2813 WARN_INSN(insn, "undefined stack state"); 2814 return 1; 2815 } 2816 return 0; 2817 } 2818 2819 if (cfi->type == UNWIND_HINT_TYPE_REGS || 2820 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) 2821 return update_cfi_state_regs(insn, cfi, op); 2822 2823 switch (op->dest.type) { 2824 2825 case OP_DEST_REG: 2826 switch (op->src.type) { 2827 2828 case OP_SRC_REG: 2829 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP && 2830 cfa->base == CFI_SP && 2831 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) { 2832 2833 /* mov %rsp, %rbp */ 2834 cfa->base = op->dest.reg; 2835 cfi->bp_scratch = false; 2836 } 2837 2838 else if (op->src.reg == CFI_SP && 2839 op->dest.reg == CFI_BP && cfi->drap) { 2840 2841 /* drap: mov %rsp, %rbp */ 2842 regs[CFI_BP].base = CFI_BP; 2843 regs[CFI_BP].offset = -cfi->stack_size; 2844 cfi->bp_scratch = false; 2845 } 2846 2847 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2848 2849 /* 2850 * mov %rsp, %reg 2851 * 2852 * This is needed for the rare case where GCC 2853 * does: 2854 * 2855 * mov %rsp, %rax 2856 * ... 2857 * mov %rax, %rsp 2858 */ 2859 cfi->vals[op->dest.reg].base = CFI_CFA; 2860 cfi->vals[op->dest.reg].offset = -cfi->stack_size; 2861 } 2862 2863 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP && 2864 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) { 2865 2866 /* 2867 * mov %rbp, %rsp 2868 * 2869 * Restore the original stack pointer (Clang). 2870 */ 2871 cfi->stack_size = -cfi->regs[CFI_BP].offset; 2872 } 2873 2874 else if (op->dest.reg == cfa->base) { 2875 2876 /* mov %reg, %rsp */ 2877 if (cfa->base == CFI_SP && 2878 cfi->vals[op->src.reg].base == CFI_CFA) { 2879 2880 /* 2881 * This is needed for the rare case 2882 * where GCC does something dumb like: 2883 * 2884 * lea 0x8(%rsp), %rcx 2885 * ... 2886 * mov %rcx, %rsp 2887 */ 2888 cfa->offset = -cfi->vals[op->src.reg].offset; 2889 cfi->stack_size = cfa->offset; 2890 2891 } else if (cfa->base == CFI_SP && 2892 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2893 cfi->vals[op->src.reg].offset == cfa->offset) { 2894 2895 /* 2896 * Stack swizzle: 2897 * 2898 * 1: mov %rsp, (%[tos]) 2899 * 2: mov %[tos], %rsp 2900 * ... 2901 * 3: pop %rsp 2902 * 2903 * Where: 2904 * 2905 * 1 - places a pointer to the previous 2906 * stack at the Top-of-Stack of the 2907 * new stack. 2908 * 2909 * 2 - switches to the new stack. 2910 * 2911 * 3 - pops the Top-of-Stack to restore 2912 * the original stack. 2913 * 2914 * Note: we set base to SP_INDIRECT 2915 * here and preserve offset. Therefore 2916 * when the unwinder reaches ToS it 2917 * will dereference SP and then add the 2918 * offset to find the next frame, IOW: 2919 * (%rsp) + offset. 2920 */ 2921 cfa->base = CFI_SP_INDIRECT; 2922 2923 } else { 2924 cfa->base = CFI_UNDEFINED; 2925 cfa->offset = 0; 2926 } 2927 } 2928 2929 else if (op->dest.reg == CFI_SP && 2930 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2931 cfi->vals[op->src.reg].offset == cfa->offset) { 2932 2933 /* 2934 * The same stack swizzle case 2) as above. But 2935 * because we can't change cfa->base, case 3) 2936 * will become a regular POP. Pretend we're a 2937 * PUSH so things don't go unbalanced. 2938 */ 2939 cfi->stack_size += 8; 2940 } 2941 2942 2943 break; 2944 2945 case OP_SRC_ADD: 2946 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) { 2947 2948 /* add imm, %rsp */ 2949 cfi->stack_size -= op->src.offset; 2950 if (cfa->base == CFI_SP) 2951 cfa->offset -= op->src.offset; 2952 break; 2953 } 2954 2955 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP && 2956 insn->sym->frame_pointer) { 2957 /* addi.d fp,sp,imm on LoongArch */ 2958 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) { 2959 cfa->base = CFI_BP; 2960 cfa->offset = 0; 2961 } 2962 break; 2963 } 2964 2965 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { 2966 /* addi.d sp,fp,imm on LoongArch */ 2967 if (cfa->base == CFI_BP && cfa->offset == 0) { 2968 if (insn->sym->frame_pointer) { 2969 cfa->base = CFI_SP; 2970 cfa->offset = -op->src.offset; 2971 } 2972 } else { 2973 /* lea disp(%rbp), %rsp */ 2974 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); 2975 } 2976 break; 2977 } 2978 2979 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2980 2981 /* drap: lea disp(%rsp), %drap */ 2982 cfi->drap_reg = op->dest.reg; 2983 2984 /* 2985 * lea disp(%rsp), %reg 2986 * 2987 * This is needed for the rare case where GCC 2988 * does something dumb like: 2989 * 2990 * lea 0x8(%rsp), %rcx 2991 * ... 2992 * mov %rcx, %rsp 2993 */ 2994 cfi->vals[op->dest.reg].base = CFI_CFA; 2995 cfi->vals[op->dest.reg].offset = \ 2996 -cfi->stack_size + op->src.offset; 2997 2998 break; 2999 } 3000 3001 if (cfi->drap && op->dest.reg == CFI_SP && 3002 op->src.reg == cfi->drap_reg) { 3003 3004 /* drap: lea disp(%drap), %rsp */ 3005 cfa->base = CFI_SP; 3006 cfa->offset = cfi->stack_size = -op->src.offset; 3007 cfi->drap_reg = CFI_UNDEFINED; 3008 cfi->drap = false; 3009 break; 3010 } 3011 3012 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { 3013 WARN_INSN(insn, "unsupported stack register modification"); 3014 return -1; 3015 } 3016 3017 break; 3018 3019 case OP_SRC_AND: 3020 if (op->dest.reg != CFI_SP || 3021 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || 3022 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { 3023 WARN_INSN(insn, "unsupported stack pointer realignment"); 3024 return -1; 3025 } 3026 3027 if (cfi->drap_reg != CFI_UNDEFINED) { 3028 /* drap: and imm, %rsp */ 3029 cfa->base = cfi->drap_reg; 3030 cfa->offset = cfi->stack_size = 0; 3031 cfi->drap = true; 3032 } 3033 3034 /* 3035 * Older versions of GCC (4.8ish) realign the stack 3036 * without DRAP, with a frame pointer. 3037 */ 3038 3039 break; 3040 3041 case OP_SRC_POP: 3042 case OP_SRC_POPF: 3043 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) { 3044 3045 /* pop %rsp; # restore from a stack swizzle */ 3046 cfa->base = CFI_SP; 3047 break; 3048 } 3049 3050 if (!cfi->drap && op->dest.reg == cfa->base) { 3051 3052 /* pop %rbp */ 3053 cfa->base = CFI_SP; 3054 } 3055 3056 if (cfi->drap && cfa->base == CFI_BP_INDIRECT && 3057 op->dest.reg == cfi->drap_reg && 3058 cfi->drap_offset == -cfi->stack_size) { 3059 3060 /* drap: pop %drap */ 3061 cfa->base = cfi->drap_reg; 3062 cfa->offset = 0; 3063 cfi->drap_offset = -1; 3064 3065 } else if (cfi->stack_size == -regs[op->dest.reg].offset) { 3066 3067 /* pop %reg */ 3068 restore_reg(cfi, op->dest.reg); 3069 } 3070 3071 cfi->stack_size -= 8; 3072 if (cfa->base == CFI_SP) 3073 cfa->offset -= 8; 3074 3075 break; 3076 3077 case OP_SRC_REG_INDIRECT: 3078 if (!cfi->drap && op->dest.reg == cfa->base && 3079 op->dest.reg == CFI_BP) { 3080 3081 /* mov disp(%rsp), %rbp */ 3082 cfa->base = CFI_SP; 3083 cfa->offset = cfi->stack_size; 3084 } 3085 3086 if (cfi->drap && op->src.reg == CFI_BP && 3087 op->src.offset == cfi->drap_offset) { 3088 3089 /* drap: mov disp(%rbp), %drap */ 3090 cfa->base = cfi->drap_reg; 3091 cfa->offset = 0; 3092 cfi->drap_offset = -1; 3093 } 3094 3095 if (cfi->drap && op->src.reg == CFI_BP && 3096 op->src.offset == regs[op->dest.reg].offset) { 3097 3098 /* drap: mov disp(%rbp), %reg */ 3099 restore_reg(cfi, op->dest.reg); 3100 3101 } else if (op->src.reg == cfa->base && 3102 op->src.offset == regs[op->dest.reg].offset + cfa->offset) { 3103 3104 /* mov disp(%rbp), %reg */ 3105 /* mov disp(%rsp), %reg */ 3106 restore_reg(cfi, op->dest.reg); 3107 3108 } else if (op->src.reg == CFI_SP && 3109 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) { 3110 3111 /* mov disp(%rsp), %reg */ 3112 restore_reg(cfi, op->dest.reg); 3113 } 3114 3115 break; 3116 3117 default: 3118 WARN_INSN(insn, "unknown stack-related instruction"); 3119 return -1; 3120 } 3121 3122 break; 3123 3124 case OP_DEST_PUSH: 3125 case OP_DEST_PUSHF: 3126 cfi->stack_size += 8; 3127 if (cfa->base == CFI_SP) 3128 cfa->offset += 8; 3129 3130 if (op->src.type != OP_SRC_REG) 3131 break; 3132 3133 if (cfi->drap) { 3134 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3135 3136 /* drap: push %drap */ 3137 cfa->base = CFI_BP_INDIRECT; 3138 cfa->offset = -cfi->stack_size; 3139 3140 /* save drap so we know when to restore it */ 3141 cfi->drap_offset = -cfi->stack_size; 3142 3143 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) { 3144 3145 /* drap: push %rbp */ 3146 cfi->stack_size = 0; 3147 3148 } else { 3149 3150 /* drap: push %reg */ 3151 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); 3152 } 3153 3154 } else { 3155 3156 /* push %reg */ 3157 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size); 3158 } 3159 3160 /* detect when asm code uses rbp as a scratch register */ 3161 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP && 3162 cfa->base != CFI_BP) 3163 cfi->bp_scratch = true; 3164 break; 3165 3166 case OP_DEST_REG_INDIRECT: 3167 3168 if (cfi->drap) { 3169 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3170 3171 /* drap: mov %drap, disp(%rbp) */ 3172 cfa->base = CFI_BP_INDIRECT; 3173 cfa->offset = op->dest.offset; 3174 3175 /* save drap offset so we know when to restore it */ 3176 cfi->drap_offset = op->dest.offset; 3177 } else { 3178 3179 /* drap: mov reg, disp(%rbp) */ 3180 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); 3181 } 3182 3183 } else if (op->dest.reg == cfa->base) { 3184 3185 /* mov reg, disp(%rbp) */ 3186 /* mov reg, disp(%rsp) */ 3187 save_reg(cfi, op->src.reg, CFI_CFA, 3188 op->dest.offset - cfi->cfa.offset); 3189 3190 } else if (op->dest.reg == CFI_SP) { 3191 3192 /* mov reg, disp(%rsp) */ 3193 save_reg(cfi, op->src.reg, CFI_CFA, 3194 op->dest.offset - cfi->stack_size); 3195 3196 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) { 3197 3198 /* mov %rsp, (%reg); # setup a stack swizzle. */ 3199 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT; 3200 cfi->vals[op->dest.reg].offset = cfa->offset; 3201 } 3202 3203 break; 3204 3205 case OP_DEST_MEM: 3206 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { 3207 WARN_INSN(insn, "unknown stack-related memory operation"); 3208 return -1; 3209 } 3210 3211 /* pop mem */ 3212 cfi->stack_size -= 8; 3213 if (cfa->base == CFI_SP) 3214 cfa->offset -= 8; 3215 3216 break; 3217 3218 default: 3219 WARN_INSN(insn, "unknown stack-related instruction"); 3220 return -1; 3221 } 3222 3223 return 0; 3224 } 3225 3226 /* 3227 * The stack layouts of alternatives instructions can sometimes diverge when 3228 * they have stack modifications. That's fine as long as the potential stack 3229 * layouts don't conflict at any given potential instruction boundary. 3230 * 3231 * Flatten the CFIs of the different alternative code streams (both original 3232 * and replacement) into a single shared CFI array which can be used to detect 3233 * conflicts and nicely feed a linear array of ORC entries to the unwinder. 3234 */ 3235 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn) 3236 { 3237 struct cfi_state **alt_cfi; 3238 int group_off; 3239 3240 if (!insn->alt_group) 3241 return 0; 3242 3243 if (!insn->cfi) { 3244 WARN("CFI missing"); 3245 return -1; 3246 } 3247 3248 alt_cfi = insn->alt_group->cfi; 3249 group_off = insn->offset - insn->alt_group->first_insn->offset; 3250 3251 if (!alt_cfi[group_off]) { 3252 alt_cfi[group_off] = insn->cfi; 3253 } else { 3254 if (cficmp(alt_cfi[group_off], insn->cfi)) { 3255 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group; 3256 struct instruction *orig = orig_group->first_insn; 3257 WARN_INSN(orig, "stack layout conflict in alternatives: %s", 3258 offstr(insn->sec, insn->offset)); 3259 return -1; 3260 } 3261 } 3262 3263 return 0; 3264 } 3265 3266 static int handle_insn_ops(struct instruction *insn, 3267 struct instruction *next_insn, 3268 struct insn_state *state) 3269 { 3270 struct stack_op *op; 3271 int ret; 3272 3273 for (op = insn->stack_ops; op; op = op->next) { 3274 3275 ret = update_cfi_state(insn, next_insn, &state->cfi, op); 3276 if (ret) 3277 return ret; 3278 3279 if (!opts.uaccess || !insn->alt_group) 3280 continue; 3281 3282 if (op->dest.type == OP_DEST_PUSHF) { 3283 if (!state->uaccess_stack) { 3284 state->uaccess_stack = 1; 3285 } else if (state->uaccess_stack >> 31) { 3286 WARN_INSN(insn, "PUSHF stack exhausted"); 3287 return 1; 3288 } 3289 state->uaccess_stack <<= 1; 3290 state->uaccess_stack |= state->uaccess; 3291 } 3292 3293 if (op->src.type == OP_SRC_POPF) { 3294 if (state->uaccess_stack) { 3295 state->uaccess = state->uaccess_stack & 1; 3296 state->uaccess_stack >>= 1; 3297 if (state->uaccess_stack == 1) 3298 state->uaccess_stack = 0; 3299 } 3300 } 3301 } 3302 3303 return 0; 3304 } 3305 3306 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) 3307 { 3308 struct cfi_state *cfi1 = insn->cfi; 3309 int i; 3310 3311 if (!cfi1) { 3312 WARN("CFI missing"); 3313 return false; 3314 } 3315 3316 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { 3317 3318 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d", 3319 cfi1->cfa.base, cfi1->cfa.offset, 3320 cfi2->cfa.base, cfi2->cfa.offset); 3321 return false; 3322 3323 } 3324 3325 if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) { 3326 for (i = 0; i < CFI_NUM_REGS; i++) { 3327 3328 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], sizeof(struct cfi_reg))) 3329 continue; 3330 3331 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", 3332 i, cfi1->regs[i].base, cfi1->regs[i].offset, 3333 i, cfi2->regs[i].base, cfi2->regs[i].offset); 3334 } 3335 return false; 3336 } 3337 3338 if (cfi1->type != cfi2->type) { 3339 3340 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d", 3341 cfi1->type, cfi2->type); 3342 return false; 3343 } 3344 3345 if (cfi1->drap != cfi2->drap || 3346 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || 3347 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { 3348 3349 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", 3350 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, 3351 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); 3352 return false; 3353 } 3354 3355 return true; 3356 } 3357 3358 static inline bool func_uaccess_safe(struct symbol *func) 3359 { 3360 if (func) 3361 return func->uaccess_safe; 3362 3363 return false; 3364 } 3365 3366 static inline const char *call_dest_name(struct instruction *insn) 3367 { 3368 static char pvname[19]; 3369 struct reloc *reloc; 3370 int idx; 3371 3372 if (insn_call_dest(insn)) 3373 return insn_call_dest(insn)->name; 3374 3375 reloc = insn_reloc(NULL, insn); 3376 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) { 3377 idx = (reloc_addend(reloc) / sizeof(void *)); 3378 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx); 3379 return pvname; 3380 } 3381 3382 return "{dynamic}"; 3383 } 3384 3385 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn) 3386 { 3387 struct symbol *target; 3388 struct reloc *reloc; 3389 int idx; 3390 3391 reloc = insn_reloc(file, insn); 3392 if (!reloc || strcmp(reloc->sym->name, "pv_ops")) 3393 return false; 3394 3395 idx = arch_insn_adjusted_addend(insn, reloc) / sizeof(void *); 3396 3397 if (file->pv_ops[idx].clean) 3398 return true; 3399 3400 file->pv_ops[idx].clean = true; 3401 3402 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) { 3403 if (!target->sec->noinstr) { 3404 WARN("pv_ops[%d]: %s", idx, target->name); 3405 file->pv_ops[idx].clean = false; 3406 } 3407 } 3408 3409 return file->pv_ops[idx].clean; 3410 } 3411 3412 static inline bool noinstr_call_dest(struct objtool_file *file, 3413 struct instruction *insn, 3414 struct symbol *func) 3415 { 3416 /* 3417 * We can't deal with indirect function calls at present; 3418 * assume they're instrumented. 3419 */ 3420 if (!func) { 3421 if (file->pv_ops) 3422 return pv_call_dest(file, insn); 3423 3424 return false; 3425 } 3426 3427 /* 3428 * If the symbol is from a noinstr section; we good. 3429 */ 3430 if (func->sec->noinstr) 3431 return true; 3432 3433 /* 3434 * If the symbol is a static_call trampoline, we can't tell. 3435 */ 3436 if (func->static_call_tramp) 3437 return true; 3438 3439 /* 3440 * The __ubsan_handle_*() calls are like WARN(), they only happen when 3441 * something 'BAD' happened. At the risk of taking the machine down, 3442 * let them proceed to get the message out. 3443 */ 3444 if (!strncmp(func->name, "__ubsan_handle_", 15)) 3445 return true; 3446 3447 return false; 3448 } 3449 3450 static int validate_call(struct objtool_file *file, 3451 struct instruction *insn, 3452 struct insn_state *state) 3453 { 3454 if (state->noinstr && state->instr <= 0 && 3455 !noinstr_call_dest(file, insn, insn_call_dest(insn))) { 3456 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn)); 3457 return 1; 3458 } 3459 3460 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) { 3461 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn)); 3462 return 1; 3463 } 3464 3465 if (state->df) { 3466 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn)); 3467 return 1; 3468 } 3469 3470 return 0; 3471 } 3472 3473 static int validate_sibling_call(struct objtool_file *file, 3474 struct instruction *insn, 3475 struct insn_state *state) 3476 { 3477 if (insn_func(insn) && has_modified_stack_frame(insn, state)) { 3478 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame"); 3479 return 1; 3480 } 3481 3482 return validate_call(file, insn, state); 3483 } 3484 3485 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 3486 { 3487 if (state->noinstr && state->instr > 0) { 3488 WARN_INSN(insn, "return with instrumentation enabled"); 3489 return 1; 3490 } 3491 3492 if (state->uaccess && !func_uaccess_safe(func)) { 3493 WARN_INSN(insn, "return with UACCESS enabled"); 3494 return 1; 3495 } 3496 3497 if (!state->uaccess && func_uaccess_safe(func)) { 3498 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function"); 3499 return 1; 3500 } 3501 3502 if (state->df) { 3503 WARN_INSN(insn, "return with DF set"); 3504 return 1; 3505 } 3506 3507 if (func && has_modified_stack_frame(insn, state)) { 3508 WARN_INSN(insn, "return with modified stack frame"); 3509 return 1; 3510 } 3511 3512 if (state->cfi.bp_scratch) { 3513 WARN_INSN(insn, "BP used as a scratch register"); 3514 return 1; 3515 } 3516 3517 return 0; 3518 } 3519 3520 static struct instruction *next_insn_to_validate(struct objtool_file *file, 3521 struct instruction *insn) 3522 { 3523 struct alt_group *alt_group = insn->alt_group; 3524 3525 /* 3526 * Simulate the fact that alternatives are patched in-place. When the 3527 * end of a replacement alt_group is reached, redirect objtool flow to 3528 * the end of the original alt_group. 3529 * 3530 * insn->alts->insn -> alt_group->first_insn 3531 * ... 3532 * alt_group->last_insn 3533 * [alt_group->nop] -> next(orig_group->last_insn) 3534 */ 3535 if (alt_group) { 3536 if (alt_group->nop) { 3537 /* ->nop implies ->orig_group */ 3538 if (insn == alt_group->last_insn) 3539 return alt_group->nop; 3540 if (insn == alt_group->nop) 3541 goto next_orig; 3542 } 3543 if (insn == alt_group->last_insn && alt_group->orig_group) 3544 goto next_orig; 3545 } 3546 3547 return next_insn_same_sec(file, insn); 3548 3549 next_orig: 3550 return next_insn_same_sec(file, alt_group->orig_group->last_insn); 3551 } 3552 3553 static bool skip_alt_group(struct instruction *insn) 3554 { 3555 struct instruction *alt_insn = insn->alts ? insn->alts->insn : NULL; 3556 3557 if (!insn->alt_group) 3558 return false; 3559 3560 /* ANNOTATE_IGNORE_ALTERNATIVE */ 3561 if (insn->alt_group->ignore) 3562 return true; 3563 3564 /* 3565 * For NOP patched with CLAC/STAC, only follow the latter to avoid 3566 * impossible code paths combining patched CLAC with unpatched STAC 3567 * or vice versa. 3568 * 3569 * ANNOTATE_IGNORE_ALTERNATIVE could have been used here, but Linus 3570 * requested not to do that to avoid hurting .s file readability 3571 * around CLAC/STAC alternative sites. 3572 */ 3573 3574 if (!alt_insn) 3575 return false; 3576 3577 /* Don't override ASM_{CLAC,STAC}_UNSAFE */ 3578 if (alt_insn->alt_group && alt_insn->alt_group->ignore) 3579 return false; 3580 3581 return alt_insn->type == INSN_CLAC || alt_insn->type == INSN_STAC; 3582 } 3583 3584 static int checksum_debug_init(struct objtool_file *file) 3585 { 3586 char *dup, *s; 3587 3588 if (!opts.debug_checksum) 3589 return 0; 3590 3591 dup = strdup(opts.debug_checksum); 3592 if (!dup) { 3593 ERROR_GLIBC("strdup"); 3594 return -1; 3595 } 3596 3597 s = dup; 3598 while (*s) { 3599 struct symbol *func; 3600 char *comma; 3601 3602 comma = strchr(s, ','); 3603 if (comma) 3604 *comma = '\0'; 3605 3606 func = find_symbol_by_name(file->elf, s); 3607 if (!func || !is_func_sym(func)) 3608 WARN("--debug-checksum: can't find '%s'", s); 3609 else 3610 func->debug_checksum = 1; 3611 3612 if (!comma) 3613 break; 3614 3615 s = comma + 1; 3616 } 3617 3618 free(dup); 3619 return 0; 3620 } 3621 3622 static void checksum_update_insn(struct objtool_file *file, struct symbol *func, 3623 struct instruction *insn) 3624 { 3625 struct reloc *reloc = insn_reloc(file, insn); 3626 unsigned long offset; 3627 struct symbol *sym; 3628 3629 if (insn->fake) 3630 return; 3631 3632 checksum_update(func, insn, insn->sec->data->d_buf + insn->offset, insn->len); 3633 3634 if (!reloc) { 3635 struct symbol *call_dest = insn_call_dest(insn); 3636 3637 if (call_dest) 3638 checksum_update(func, insn, call_dest->demangled_name, 3639 strlen(call_dest->demangled_name)); 3640 return; 3641 } 3642 3643 sym = reloc->sym; 3644 offset = arch_insn_adjusted_addend(insn, reloc); 3645 3646 if (is_string_sec(sym->sec)) { 3647 char *str; 3648 3649 str = sym->sec->data->d_buf + sym->offset + offset; 3650 checksum_update(func, insn, str, strlen(str)); 3651 return; 3652 } 3653 3654 if (is_sec_sym(sym)) { 3655 sym = find_symbol_containing(reloc->sym->sec, offset); 3656 if (!sym) 3657 return; 3658 3659 offset -= sym->offset; 3660 } 3661 3662 checksum_update(func, insn, sym->demangled_name, strlen(sym->demangled_name)); 3663 checksum_update(func, insn, &offset, sizeof(offset)); 3664 } 3665 3666 /* 3667 * Follow the branch starting at the given instruction, and recursively follow 3668 * any other branches (jumps). Meanwhile, track the frame pointer state at 3669 * each instruction and validate all the rules described in 3670 * tools/objtool/Documentation/objtool.txt. 3671 */ 3672 static int validate_branch(struct objtool_file *file, struct symbol *func, 3673 struct instruction *insn, struct insn_state state) 3674 { 3675 struct alternative *alt; 3676 struct instruction *next_insn, *prev_insn = NULL; 3677 u8 visited; 3678 int ret; 3679 3680 if (func && func->ignore) 3681 return 0; 3682 3683 while (1) { 3684 next_insn = next_insn_to_validate(file, insn); 3685 3686 if (opts.checksum && func && insn->sec) 3687 checksum_update_insn(file, func, insn); 3688 3689 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { 3690 /* Ignore KCFI type preambles, which always fall through */ 3691 if (is_prefix_func(func)) 3692 return 0; 3693 3694 if (file->ignore_unreachables) 3695 return 0; 3696 3697 WARN("%s() falls through to next function %s()", 3698 func->name, insn_func(insn)->name); 3699 func->warned = 1; 3700 3701 return 1; 3702 } 3703 3704 visited = VISITED_BRANCH << state.uaccess; 3705 if (insn->visited & VISITED_BRANCH_MASK) { 3706 if (!insn->hint && !insn_cfi_match(insn, &state.cfi)) 3707 return 1; 3708 3709 if (insn->visited & visited) 3710 return 0; 3711 } else { 3712 nr_insns_visited++; 3713 } 3714 3715 if (state.noinstr) 3716 state.instr += insn->instr; 3717 3718 if (insn->hint) { 3719 if (insn->restore) { 3720 struct instruction *save_insn, *i; 3721 3722 i = insn; 3723 save_insn = NULL; 3724 3725 sym_for_each_insn_continue_reverse(file, func, i) { 3726 if (i->save) { 3727 save_insn = i; 3728 break; 3729 } 3730 } 3731 3732 if (!save_insn) { 3733 WARN_INSN(insn, "no corresponding CFI save for CFI restore"); 3734 return 1; 3735 } 3736 3737 if (!save_insn->visited) { 3738 /* 3739 * If the restore hint insn is at the 3740 * beginning of a basic block and was 3741 * branched to from elsewhere, and the 3742 * save insn hasn't been visited yet, 3743 * defer following this branch for now. 3744 * It will be seen later via the 3745 * straight-line path. 3746 */ 3747 if (!prev_insn) 3748 return 0; 3749 3750 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo"); 3751 return 1; 3752 } 3753 3754 insn->cfi = save_insn->cfi; 3755 nr_cfi_reused++; 3756 } 3757 3758 state.cfi = *insn->cfi; 3759 } else { 3760 /* XXX track if we actually changed state.cfi */ 3761 3762 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) { 3763 insn->cfi = prev_insn->cfi; 3764 nr_cfi_reused++; 3765 } else { 3766 insn->cfi = cfi_hash_find_or_add(&state.cfi); 3767 } 3768 } 3769 3770 insn->visited |= visited; 3771 3772 if (propagate_alt_cfi(file, insn)) 3773 return 1; 3774 3775 if (insn->alts) { 3776 for (alt = insn->alts; alt; alt = alt->next) { 3777 ret = validate_branch(file, func, alt->insn, state); 3778 if (ret) { 3779 BT_INSN(insn, "(alt)"); 3780 return ret; 3781 } 3782 } 3783 } 3784 3785 if (skip_alt_group(insn)) 3786 return 0; 3787 3788 if (handle_insn_ops(insn, next_insn, &state)) 3789 return 1; 3790 3791 switch (insn->type) { 3792 3793 case INSN_RETURN: 3794 return validate_return(func, insn, &state); 3795 3796 case INSN_CALL: 3797 case INSN_CALL_DYNAMIC: 3798 ret = validate_call(file, insn, &state); 3799 if (ret) 3800 return ret; 3801 3802 if (opts.stackval && func && !is_special_call(insn) && 3803 !has_valid_stack_frame(&state)) { 3804 WARN_INSN(insn, "call without frame pointer save/setup"); 3805 return 1; 3806 } 3807 3808 break; 3809 3810 case INSN_JUMP_CONDITIONAL: 3811 case INSN_JUMP_UNCONDITIONAL: 3812 if (is_sibling_call(insn)) { 3813 ret = validate_sibling_call(file, insn, &state); 3814 if (ret) 3815 return ret; 3816 3817 } else if (insn->jump_dest) { 3818 ret = validate_branch(file, func, 3819 insn->jump_dest, state); 3820 if (ret) { 3821 BT_INSN(insn, "(branch)"); 3822 return ret; 3823 } 3824 } 3825 3826 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3827 return 0; 3828 3829 break; 3830 3831 case INSN_JUMP_DYNAMIC: 3832 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3833 if (is_sibling_call(insn)) { 3834 ret = validate_sibling_call(file, insn, &state); 3835 if (ret) 3836 return ret; 3837 } 3838 3839 if (insn->type == INSN_JUMP_DYNAMIC) 3840 return 0; 3841 3842 break; 3843 3844 case INSN_SYSCALL: 3845 if (func && (!next_insn || !next_insn->hint)) { 3846 WARN_INSN(insn, "unsupported instruction in callable function"); 3847 return 1; 3848 } 3849 3850 break; 3851 3852 case INSN_SYSRET: 3853 if (func && (!next_insn || !next_insn->hint)) { 3854 WARN_INSN(insn, "unsupported instruction in callable function"); 3855 return 1; 3856 } 3857 3858 return 0; 3859 3860 case INSN_STAC: 3861 if (!opts.uaccess) 3862 break; 3863 3864 if (state.uaccess) { 3865 WARN_INSN(insn, "recursive UACCESS enable"); 3866 return 1; 3867 } 3868 3869 state.uaccess = true; 3870 break; 3871 3872 case INSN_CLAC: 3873 if (!opts.uaccess) 3874 break; 3875 3876 if (!state.uaccess && func) { 3877 WARN_INSN(insn, "redundant UACCESS disable"); 3878 return 1; 3879 } 3880 3881 if (func_uaccess_safe(func) && !state.uaccess_stack) { 3882 WARN_INSN(insn, "UACCESS-safe disables UACCESS"); 3883 return 1; 3884 } 3885 3886 state.uaccess = false; 3887 break; 3888 3889 case INSN_STD: 3890 if (state.df) { 3891 WARN_INSN(insn, "recursive STD"); 3892 return 1; 3893 } 3894 3895 state.df = true; 3896 break; 3897 3898 case INSN_CLD: 3899 if (!state.df && func) { 3900 WARN_INSN(insn, "redundant CLD"); 3901 return 1; 3902 } 3903 3904 state.df = false; 3905 break; 3906 3907 default: 3908 break; 3909 } 3910 3911 if (insn->dead_end) 3912 return 0; 3913 3914 if (!next_insn) { 3915 if (state.cfi.cfa.base == CFI_UNDEFINED) 3916 return 0; 3917 if (file->ignore_unreachables) 3918 return 0; 3919 3920 WARN("%s%sunexpected end of section %s", 3921 func ? func->name : "", func ? "(): " : "", 3922 insn->sec->name); 3923 return 1; 3924 } 3925 3926 prev_insn = insn; 3927 insn = next_insn; 3928 } 3929 3930 return 0; 3931 } 3932 3933 static int validate_unwind_hint(struct objtool_file *file, 3934 struct instruction *insn, 3935 struct insn_state *state) 3936 { 3937 if (insn->hint && !insn->visited) { 3938 struct symbol *func = insn_func(insn); 3939 int ret; 3940 3941 if (opts.checksum) 3942 checksum_init(func); 3943 3944 ret = validate_branch(file, func, insn, *state); 3945 if (ret) 3946 BT_INSN(insn, "<=== (hint)"); 3947 return ret; 3948 } 3949 3950 return 0; 3951 } 3952 3953 static int validate_unwind_hints(struct objtool_file *file, struct section *sec) 3954 { 3955 struct instruction *insn; 3956 struct insn_state state; 3957 int warnings = 0; 3958 3959 if (!file->hints) 3960 return 0; 3961 3962 init_insn_state(file, &state, sec); 3963 3964 if (sec) { 3965 sec_for_each_insn(file, sec, insn) 3966 warnings += validate_unwind_hint(file, insn, &state); 3967 } else { 3968 for_each_insn(file, insn) 3969 warnings += validate_unwind_hint(file, insn, &state); 3970 } 3971 3972 return warnings; 3973 } 3974 3975 /* 3976 * Validate rethunk entry constraint: must untrain RET before the first RET. 3977 * 3978 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes 3979 * before an actual RET instruction. 3980 */ 3981 static int validate_unret(struct objtool_file *file, struct instruction *insn) 3982 { 3983 struct instruction *next, *dest; 3984 int ret; 3985 3986 for (;;) { 3987 next = next_insn_to_validate(file, insn); 3988 3989 if (insn->visited & VISITED_UNRET) 3990 return 0; 3991 3992 insn->visited |= VISITED_UNRET; 3993 3994 if (insn->alts) { 3995 struct alternative *alt; 3996 for (alt = insn->alts; alt; alt = alt->next) { 3997 ret = validate_unret(file, alt->insn); 3998 if (ret) { 3999 BT_INSN(insn, "(alt)"); 4000 return ret; 4001 } 4002 } 4003 } 4004 4005 switch (insn->type) { 4006 4007 case INSN_CALL_DYNAMIC: 4008 case INSN_JUMP_DYNAMIC: 4009 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4010 WARN_INSN(insn, "early indirect call"); 4011 return 1; 4012 4013 case INSN_JUMP_UNCONDITIONAL: 4014 case INSN_JUMP_CONDITIONAL: 4015 if (!is_sibling_call(insn)) { 4016 if (!insn->jump_dest) { 4017 WARN_INSN(insn, "unresolved jump target after linking?!?"); 4018 return 1; 4019 } 4020 ret = validate_unret(file, insn->jump_dest); 4021 if (ret) { 4022 BT_INSN(insn, "(branch%s)", 4023 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : ""); 4024 return ret; 4025 } 4026 4027 if (insn->type == INSN_JUMP_UNCONDITIONAL) 4028 return 0; 4029 4030 break; 4031 } 4032 4033 /* fallthrough */ 4034 case INSN_CALL: 4035 dest = find_insn(file, insn_call_dest(insn)->sec, 4036 insn_call_dest(insn)->offset); 4037 if (!dest) { 4038 WARN("Unresolved function after linking!?: %s", 4039 insn_call_dest(insn)->name); 4040 return 1; 4041 } 4042 4043 ret = validate_unret(file, dest); 4044 if (ret) { 4045 BT_INSN(insn, "(call)"); 4046 return ret; 4047 } 4048 /* 4049 * If a call returns without error, it must have seen UNTRAIN_RET. 4050 * Therefore any non-error return is a success. 4051 */ 4052 return 0; 4053 4054 case INSN_RETURN: 4055 WARN_INSN(insn, "RET before UNTRAIN"); 4056 return 1; 4057 4058 case INSN_SYSCALL: 4059 break; 4060 4061 case INSN_SYSRET: 4062 return 0; 4063 4064 case INSN_NOP: 4065 if (insn->retpoline_safe) 4066 return 0; 4067 break; 4068 4069 default: 4070 break; 4071 } 4072 4073 if (insn->dead_end) 4074 return 0; 4075 4076 if (!next) { 4077 WARN_INSN(insn, "teh end!"); 4078 return 1; 4079 } 4080 insn = next; 4081 } 4082 4083 return 0; 4084 } 4085 4086 /* 4087 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter 4088 * VALIDATE_UNRET_END before RET. 4089 */ 4090 static int validate_unrets(struct objtool_file *file) 4091 { 4092 struct instruction *insn; 4093 int warnings = 0; 4094 4095 for_each_insn(file, insn) { 4096 if (!insn->unret) 4097 continue; 4098 4099 warnings += validate_unret(file, insn); 4100 } 4101 4102 return warnings; 4103 } 4104 4105 static int validate_retpoline(struct objtool_file *file) 4106 { 4107 struct instruction *insn; 4108 int warnings = 0; 4109 4110 for_each_insn(file, insn) { 4111 if (insn->type != INSN_JUMP_DYNAMIC && 4112 insn->type != INSN_CALL_DYNAMIC && 4113 insn->type != INSN_RETURN) 4114 continue; 4115 4116 if (insn->retpoline_safe) 4117 continue; 4118 4119 if (insn->sec->init) 4120 continue; 4121 4122 if (insn->type == INSN_RETURN) { 4123 if (opts.rethunk) { 4124 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build"); 4125 warnings++; 4126 } 4127 continue; 4128 } 4129 4130 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build", 4131 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); 4132 warnings++; 4133 } 4134 4135 if (!opts.cfi) 4136 return warnings; 4137 4138 /* 4139 * kCFI call sites look like: 4140 * 4141 * movl $(-0x12345678), %r10d 4142 * addl -4(%r11), %r10d 4143 * jz 1f 4144 * ud2 4145 * 1: cs call __x86_indirect_thunk_r11 4146 * 4147 * Verify all indirect calls are kCFI adorned by checking for the 4148 * UD2. Notably, doing __nocfi calls to regular (cfi) functions is 4149 * broken. 4150 */ 4151 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 4152 struct symbol *sym = insn->sym; 4153 4154 if (sym && (sym->type == STT_NOTYPE || 4155 sym->type == STT_FUNC) && !sym->nocfi) { 4156 struct instruction *prev = 4157 prev_insn_same_sym(file, insn); 4158 4159 if (!prev || prev->type != INSN_BUG) { 4160 WARN_INSN(insn, "no-cfi indirect call!"); 4161 warnings++; 4162 } 4163 } 4164 } 4165 4166 return warnings; 4167 } 4168 4169 static bool is_kasan_insn(struct instruction *insn) 4170 { 4171 return (insn->type == INSN_CALL && 4172 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return")); 4173 } 4174 4175 static bool is_ubsan_insn(struct instruction *insn) 4176 { 4177 return (insn->type == INSN_CALL && 4178 !strcmp(insn_call_dest(insn)->name, 4179 "__ubsan_handle_builtin_unreachable")); 4180 } 4181 4182 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) 4183 { 4184 struct symbol *func = insn_func(insn); 4185 struct instruction *prev_insn; 4186 int i; 4187 4188 if (insn->type == INSN_NOP || insn->type == INSN_TRAP || 4189 insn->hole || (func && func->ignore)) 4190 return true; 4191 4192 /* 4193 * Ignore alternative replacement instructions. This can happen 4194 * when a whitelisted function uses one of the ALTERNATIVE macros. 4195 */ 4196 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 4197 !strcmp(insn->sec->name, ".altinstr_aux")) 4198 return true; 4199 4200 if (!func) 4201 return false; 4202 4203 if (func->static_call_tramp) 4204 return true; 4205 4206 /* 4207 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees 4208 * __builtin_unreachable(). The BUG() macro has an unreachable() after 4209 * the UD2, which causes GCC's undefined trap logic to emit another UD2 4210 * (or occasionally a JMP to UD2). 4211 * 4212 * It may also insert a UD2 after calling a __noreturn function. 4213 */ 4214 prev_insn = prev_insn_same_sec(file, insn); 4215 if (prev_insn && prev_insn->dead_end && 4216 (insn->type == INSN_BUG || 4217 (insn->type == INSN_JUMP_UNCONDITIONAL && 4218 insn->jump_dest && insn->jump_dest->type == INSN_BUG))) 4219 return true; 4220 4221 /* 4222 * Check if this (or a subsequent) instruction is related to 4223 * CONFIG_UBSAN or CONFIG_KASAN. 4224 * 4225 * End the search at 5 instructions to avoid going into the weeds. 4226 */ 4227 for (i = 0; i < 5; i++) { 4228 4229 if (is_kasan_insn(insn) || is_ubsan_insn(insn)) 4230 return true; 4231 4232 if (insn->type == INSN_JUMP_UNCONDITIONAL) { 4233 if (insn->jump_dest && 4234 insn_func(insn->jump_dest) == func) { 4235 insn = insn->jump_dest; 4236 continue; 4237 } 4238 4239 break; 4240 } 4241 4242 if (insn->offset + insn->len >= func->offset + func->len) 4243 break; 4244 4245 insn = next_insn_same_sec(file, insn); 4246 } 4247 4248 return false; 4249 } 4250 4251 /* 4252 * For FineIBT or kCFI, a certain number of bytes preceding the function may be 4253 * NOPs. Those NOPs may be rewritten at runtime and executed, so give them a 4254 * proper function name: __pfx_<func>. 4255 * 4256 * The NOPs may not exist for the following cases: 4257 * 4258 * - compiler cloned functions (*.cold, *.part0, etc) 4259 * - asm functions created with inline asm or without SYM_FUNC_START() 4260 * 4261 * Also, the function may already have a prefix from a previous objtool run 4262 * (livepatch extracted functions, or manually running objtool multiple times). 4263 * 4264 * So return 0 if the NOPs are missing or the function already has a prefix 4265 * symbol. 4266 */ 4267 static int create_prefix_symbol(struct objtool_file *file, struct symbol *func) 4268 { 4269 struct instruction *insn, *prev; 4270 char name[SYM_NAME_LEN]; 4271 struct cfi_state *cfi; 4272 4273 if (!is_func_sym(func) || is_prefix_func(func) || 4274 func->cold || func->static_call_tramp) 4275 return 0; 4276 4277 if ((strlen(func->name) + sizeof("__pfx_") > SYM_NAME_LEN)) { 4278 WARN("%s: symbol name too long, can't create __pfx_ symbol", 4279 func->name); 4280 return 0; 4281 } 4282 4283 if (snprintf_check(name, SYM_NAME_LEN, "__pfx_%s", func->name)) 4284 return -1; 4285 4286 if (file->klp) { 4287 struct symbol *pfx; 4288 4289 pfx = find_symbol_by_offset(func->sec, func->offset - opts.prefix); 4290 if (pfx && is_prefix_func(pfx) && !strcmp(pfx->name, name)) 4291 return 0; 4292 } 4293 4294 insn = find_insn(file, func->sec, func->offset); 4295 if (!insn) { 4296 WARN("%s: can't find starting instruction", func->name); 4297 return -1; 4298 } 4299 4300 for (prev = prev_insn_same_sec(file, insn); 4301 prev; 4302 prev = prev_insn_same_sec(file, prev)) { 4303 u64 offset; 4304 4305 if (prev->type != INSN_NOP) 4306 return 0; 4307 4308 offset = func->offset - prev->offset; 4309 4310 if (offset > opts.prefix) 4311 return 0; 4312 4313 if (offset < opts.prefix) 4314 continue; 4315 4316 if (!elf_create_symbol(file->elf, name, func->sec, 4317 GELF_ST_BIND(func->sym.st_info), 4318 GELF_ST_TYPE(func->sym.st_info), 4319 prev->offset, opts.prefix)) 4320 return -1; 4321 4322 break; 4323 } 4324 4325 if (!prev) 4326 return 0; 4327 4328 if (!insn->cfi) { 4329 /* 4330 * This can happen if stack validation isn't enabled or the 4331 * function is annotated with STACK_FRAME_NON_STANDARD. 4332 */ 4333 return 0; 4334 } 4335 4336 /* Propagate insn->cfi to the prefix code */ 4337 cfi = cfi_hash_find_or_add(insn->cfi); 4338 for (; prev != insn; prev = next_insn_same_sec(file, prev)) 4339 prev->cfi = cfi; 4340 4341 return 0; 4342 } 4343 4344 static int create_prefix_symbols(struct objtool_file *file) 4345 { 4346 struct section *sec; 4347 struct symbol *func; 4348 4349 for_each_sec(file->elf, sec) { 4350 if (!is_text_sec(sec)) 4351 continue; 4352 4353 sec_for_each_sym(sec, func) { 4354 if (create_prefix_symbol(file, func)) 4355 return -1; 4356 } 4357 } 4358 4359 return 0; 4360 } 4361 4362 static int validate_symbol(struct objtool_file *file, struct section *sec, 4363 struct symbol *sym, struct insn_state *state) 4364 { 4365 struct instruction *insn; 4366 struct symbol *func; 4367 int ret; 4368 4369 if (!sym->len) { 4370 WARN("%s() is missing an ELF size annotation", sym->name); 4371 return 1; 4372 } 4373 4374 if (sym->pfunc != sym || sym->alias != sym) 4375 return 0; 4376 4377 insn = find_insn(file, sec, sym->offset); 4378 if (!insn || insn->visited) 4379 return 0; 4380 4381 if (opts.uaccess) 4382 state->uaccess = sym->uaccess_safe; 4383 4384 func = insn_func(insn); 4385 4386 if (opts.checksum) 4387 checksum_init(func); 4388 4389 ret = validate_branch(file, func, insn, *state); 4390 if (ret) 4391 BT_INSN(insn, "<=== (sym)"); 4392 4393 if (opts.checksum) 4394 checksum_finish(func); 4395 4396 return ret; 4397 } 4398 4399 static int validate_section(struct objtool_file *file, struct section *sec) 4400 { 4401 struct insn_state state; 4402 struct symbol *func; 4403 int warnings = 0; 4404 4405 sec_for_each_sym(sec, func) { 4406 if (!is_func_sym(func)) 4407 continue; 4408 4409 init_insn_state(file, &state, sec); 4410 set_func_state(&state.cfi); 4411 4412 warnings += validate_symbol(file, sec, func, &state); 4413 } 4414 4415 return warnings; 4416 } 4417 4418 static int validate_noinstr_sections(struct objtool_file *file) 4419 { 4420 struct section *sec; 4421 int warnings = 0; 4422 4423 sec = find_section_by_name(file->elf, ".noinstr.text"); 4424 if (sec) { 4425 warnings += validate_section(file, sec); 4426 warnings += validate_unwind_hints(file, sec); 4427 } 4428 4429 sec = find_section_by_name(file->elf, ".entry.text"); 4430 if (sec) { 4431 warnings += validate_section(file, sec); 4432 warnings += validate_unwind_hints(file, sec); 4433 } 4434 4435 sec = find_section_by_name(file->elf, ".cpuidle.text"); 4436 if (sec) { 4437 warnings += validate_section(file, sec); 4438 warnings += validate_unwind_hints(file, sec); 4439 } 4440 4441 return warnings; 4442 } 4443 4444 static int validate_functions(struct objtool_file *file) 4445 { 4446 struct section *sec; 4447 int warnings = 0; 4448 4449 for_each_sec(file->elf, sec) { 4450 if (!is_text_sec(sec)) 4451 continue; 4452 4453 warnings += validate_section(file, sec); 4454 } 4455 4456 return warnings; 4457 } 4458 4459 static void mark_endbr_used(struct instruction *insn) 4460 { 4461 if (!list_empty(&insn->call_node)) 4462 list_del_init(&insn->call_node); 4463 } 4464 4465 static bool noendbr_range(struct objtool_file *file, struct instruction *insn) 4466 { 4467 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1); 4468 struct instruction *first; 4469 4470 if (!sym) 4471 return false; 4472 4473 first = find_insn(file, sym->sec, sym->offset); 4474 if (!first) 4475 return false; 4476 4477 if (first->type != INSN_ENDBR && !first->noendbr) 4478 return false; 4479 4480 return insn->offset == sym->offset + sym->len; 4481 } 4482 4483 static int __validate_ibt_insn(struct objtool_file *file, struct instruction *insn, 4484 struct instruction *dest) 4485 { 4486 if (dest->type == INSN_ENDBR) { 4487 mark_endbr_used(dest); 4488 return 0; 4489 } 4490 4491 if (insn_func(dest) && insn_func(insn) && 4492 insn_func(dest)->pfunc == insn_func(insn)->pfunc) { 4493 /* 4494 * Anything from->to self is either _THIS_IP_ or 4495 * IRET-to-self. 4496 * 4497 * There is no sane way to annotate _THIS_IP_ since the 4498 * compiler treats the relocation as a constant and is 4499 * happy to fold in offsets, skewing any annotation we 4500 * do, leading to vast amounts of false-positives. 4501 * 4502 * There's also compiler generated _THIS_IP_ through 4503 * KCOV and such which we have no hope of annotating. 4504 * 4505 * As such, blanket accept self-references without 4506 * issue. 4507 */ 4508 return 0; 4509 } 4510 4511 /* 4512 * Accept anything ANNOTATE_NOENDBR. 4513 */ 4514 if (dest->noendbr) 4515 return 0; 4516 4517 /* 4518 * Accept if this is the instruction after a symbol 4519 * that is (no)endbr -- typical code-range usage. 4520 */ 4521 if (noendbr_range(file, dest)) 4522 return 0; 4523 4524 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4525 return 1; 4526 } 4527 4528 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn) 4529 { 4530 struct instruction *dest; 4531 struct reloc *reloc; 4532 unsigned long off; 4533 int warnings = 0; 4534 4535 /* 4536 * Looking for function pointer load relocations. Ignore 4537 * direct/indirect branches: 4538 */ 4539 switch (insn->type) { 4540 4541 case INSN_CALL: 4542 case INSN_CALL_DYNAMIC: 4543 case INSN_JUMP_CONDITIONAL: 4544 case INSN_JUMP_UNCONDITIONAL: 4545 case INSN_JUMP_DYNAMIC: 4546 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4547 case INSN_RETURN: 4548 case INSN_NOP: 4549 return 0; 4550 4551 case INSN_LEA_RIP: 4552 if (!insn_reloc(file, insn)) { 4553 /* local function pointer reference without reloc */ 4554 4555 off = arch_jump_destination(insn); 4556 4557 dest = find_insn(file, insn->sec, off); 4558 if (!dest) { 4559 WARN_INSN(insn, "corrupt function pointer reference"); 4560 return 1; 4561 } 4562 4563 return __validate_ibt_insn(file, insn, dest); 4564 } 4565 break; 4566 4567 default: 4568 break; 4569 } 4570 4571 for (reloc = insn_reloc(file, insn); 4572 reloc; 4573 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 4574 reloc_offset(reloc) + 1, 4575 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) { 4576 4577 off = reloc->sym->offset + arch_insn_adjusted_addend(insn, reloc); 4578 4579 dest = find_insn(file, reloc->sym->sec, off); 4580 if (!dest) 4581 continue; 4582 4583 warnings += __validate_ibt_insn(file, insn, dest); 4584 } 4585 4586 return warnings; 4587 } 4588 4589 static int validate_ibt_data_reloc(struct objtool_file *file, 4590 struct reloc *reloc) 4591 { 4592 struct instruction *dest; 4593 4594 dest = find_insn(file, reloc->sym->sec, 4595 reloc->sym->offset + reloc_addend(reloc)); 4596 if (!dest) 4597 return 0; 4598 4599 if (dest->type == INSN_ENDBR) { 4600 mark_endbr_used(dest); 4601 return 0; 4602 } 4603 4604 if (dest->noendbr) 4605 return 0; 4606 4607 WARN_FUNC(reloc->sec->base, reloc_offset(reloc), 4608 "data relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4609 4610 return 1; 4611 } 4612 4613 /* 4614 * Validate IBT rules and remove used ENDBR instructions from the seal list. 4615 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with 4616 * NOPs) later, in create_ibt_endbr_seal_sections(). 4617 */ 4618 static int validate_ibt(struct objtool_file *file) 4619 { 4620 struct section *sec; 4621 struct reloc *reloc; 4622 struct instruction *insn; 4623 int warnings = 0; 4624 4625 for_each_insn(file, insn) 4626 warnings += validate_ibt_insn(file, insn); 4627 4628 for_each_sec(file->elf, sec) { 4629 4630 /* Already done by validate_ibt_insn() */ 4631 if (is_text_sec(sec)) 4632 continue; 4633 4634 if (!sec->rsec) 4635 continue; 4636 4637 /* 4638 * These sections can reference text addresses, but not with 4639 * the intent to indirect branch to them. 4640 */ 4641 if ((!strncmp(sec->name, ".discard", 8) && 4642 strcmp(sec->name, ".discard.ibt_endbr_noseal")) || 4643 !strncmp(sec->name, ".debug", 6) || 4644 !strcmp(sec->name, ".altinstructions") || 4645 !strcmp(sec->name, ".ibt_endbr_seal") || 4646 !strcmp(sec->name, ".kcfi_traps") || 4647 !strcmp(sec->name, ".orc_unwind_ip") || 4648 !strcmp(sec->name, ".retpoline_sites") || 4649 !strcmp(sec->name, ".smp_locks") || 4650 !strcmp(sec->name, ".static_call_sites") || 4651 !strcmp(sec->name, "_error_injection_whitelist") || 4652 !strcmp(sec->name, "_kprobe_blacklist") || 4653 !strcmp(sec->name, "__bug_table") || 4654 !strcmp(sec->name, "__ex_table") || 4655 !strcmp(sec->name, "__jump_table") || 4656 !strcmp(sec->name, "__klp_funcs") || 4657 !strcmp(sec->name, "__mcount_loc") || 4658 !strcmp(sec->name, ".llvm.call-graph-profile") || 4659 !strcmp(sec->name, ".llvm_bb_addr_map") || 4660 !strcmp(sec->name, "__tracepoints") || 4661 !strcmp(sec->name, "__patchable_function_entries")) 4662 continue; 4663 4664 for_each_reloc(sec->rsec, reloc) 4665 warnings += validate_ibt_data_reloc(file, reloc); 4666 } 4667 4668 return warnings; 4669 } 4670 4671 static int validate_sls(struct objtool_file *file) 4672 { 4673 struct instruction *insn, *next_insn; 4674 int warnings = 0; 4675 4676 for_each_insn(file, insn) { 4677 next_insn = next_insn_same_sec(file, insn); 4678 4679 if (insn->retpoline_safe) 4680 continue; 4681 4682 switch (insn->type) { 4683 case INSN_RETURN: 4684 if (!next_insn || next_insn->type != INSN_TRAP) { 4685 WARN_INSN(insn, "missing int3 after ret"); 4686 warnings++; 4687 } 4688 4689 break; 4690 case INSN_JUMP_DYNAMIC: 4691 if (!next_insn || next_insn->type != INSN_TRAP) { 4692 WARN_INSN(insn, "missing int3 after indirect jump"); 4693 warnings++; 4694 } 4695 break; 4696 default: 4697 break; 4698 } 4699 } 4700 4701 return warnings; 4702 } 4703 4704 static int validate_reachable_instructions(struct objtool_file *file) 4705 { 4706 struct instruction *insn, *prev_insn; 4707 struct symbol *call_dest; 4708 int warnings = 0; 4709 4710 if (file->ignore_unreachables) 4711 return 0; 4712 4713 for_each_insn(file, insn) { 4714 if (insn->visited || ignore_unreachable_insn(file, insn)) 4715 continue; 4716 4717 prev_insn = prev_insn_same_sec(file, insn); 4718 if (prev_insn && prev_insn->dead_end) { 4719 call_dest = insn_call_dest(prev_insn); 4720 if (call_dest) { 4721 WARN_INSN(insn, "%s() missing __noreturn in .c/.h or NORETURN() in noreturns.h", 4722 call_dest->name); 4723 warnings++; 4724 continue; 4725 } 4726 } 4727 4728 WARN_INSN(insn, "unreachable instruction"); 4729 warnings++; 4730 } 4731 4732 return warnings; 4733 } 4734 4735 __weak bool arch_absolute_reloc(struct elf *elf, struct reloc *reloc) 4736 { 4737 unsigned int type = reloc_type(reloc); 4738 size_t sz = elf_addr_size(elf); 4739 4740 return (sz == 8) ? (type == R_ABS64) : (type == R_ABS32); 4741 } 4742 4743 static int check_abs_references(struct objtool_file *file) 4744 { 4745 struct section *sec; 4746 struct reloc *reloc; 4747 int ret = 0; 4748 4749 for_each_sec(file->elf, sec) { 4750 /* absolute references in non-loadable sections are fine */ 4751 if (!(sec->sh.sh_flags & SHF_ALLOC)) 4752 continue; 4753 4754 /* section must have an associated .rela section */ 4755 if (!sec->rsec) 4756 continue; 4757 4758 /* 4759 * Special case for compiler generated metadata that is not 4760 * consumed until after boot. 4761 */ 4762 if (!strcmp(sec->name, "__patchable_function_entries")) 4763 continue; 4764 4765 for_each_reloc(sec->rsec, reloc) { 4766 if (arch_absolute_reloc(file->elf, reloc)) { 4767 WARN("section %s has absolute relocation at offset 0x%llx", 4768 sec->name, (unsigned long long)reloc_offset(reloc)); 4769 ret++; 4770 } 4771 } 4772 } 4773 return ret; 4774 } 4775 4776 struct insn_chunk { 4777 void *addr; 4778 struct insn_chunk *next; 4779 }; 4780 4781 /* 4782 * Reduce peak RSS usage by freeing insns memory before writing the ELF file, 4783 * which can trigger more allocations for .debug_* sections whose data hasn't 4784 * been read yet. 4785 */ 4786 static void free_insns(struct objtool_file *file) 4787 { 4788 struct instruction *insn; 4789 struct insn_chunk *chunks = NULL, *chunk; 4790 4791 for_each_insn(file, insn) { 4792 if (!insn->idx) { 4793 chunk = malloc(sizeof(*chunk)); 4794 chunk->addr = insn; 4795 chunk->next = chunks; 4796 chunks = chunk; 4797 } 4798 } 4799 4800 for (chunk = chunks; chunk; chunk = chunk->next) 4801 free(chunk->addr); 4802 } 4803 4804 int check(struct objtool_file *file) 4805 { 4806 struct disas_context *disas_ctx; 4807 int ret = 0, warnings = 0; 4808 4809 arch_initial_func_cfi_state(&initial_func_cfi); 4810 init_cfi_state(&init_cfi); 4811 init_cfi_state(&func_cfi); 4812 set_func_state(&func_cfi); 4813 init_cfi_state(&force_undefined_cfi); 4814 force_undefined_cfi.force_undefined = true; 4815 4816 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) { 4817 ret = -1; 4818 goto out; 4819 } 4820 4821 cfi_hash_add(&init_cfi); 4822 cfi_hash_add(&func_cfi); 4823 4824 ret = checksum_debug_init(file); 4825 if (ret) 4826 goto out; 4827 4828 ret = decode_sections(file); 4829 if (ret) 4830 goto out; 4831 4832 if (!nr_insns) 4833 goto out; 4834 4835 if (opts.retpoline) 4836 warnings += validate_retpoline(file); 4837 4838 if (validate_branch_enabled()) { 4839 int w = 0; 4840 4841 w += validate_functions(file); 4842 w += validate_unwind_hints(file, NULL); 4843 if (!w) 4844 w += validate_reachable_instructions(file); 4845 4846 warnings += w; 4847 4848 } else if (opts.noinstr) { 4849 warnings += validate_noinstr_sections(file); 4850 } 4851 4852 if (opts.unret) { 4853 /* 4854 * Must be after validate_branch() and friends, it plays 4855 * further games with insn->visited. 4856 */ 4857 warnings += validate_unrets(file); 4858 } 4859 4860 if (opts.ibt) 4861 warnings += validate_ibt(file); 4862 4863 if (opts.sls) 4864 warnings += validate_sls(file); 4865 4866 if (opts.static_call) { 4867 ret = create_static_call_sections(file); 4868 if (ret) 4869 goto out; 4870 } 4871 4872 if (opts.retpoline) { 4873 ret = create_retpoline_sites_sections(file); 4874 if (ret) 4875 goto out; 4876 } 4877 4878 if (opts.cfi) { 4879 ret = create_cfi_sections(file); 4880 if (ret) 4881 goto out; 4882 } 4883 4884 if (opts.rethunk) { 4885 ret = create_return_sites_sections(file); 4886 if (ret) 4887 goto out; 4888 4889 if (opts.hack_skylake) { 4890 ret = create_direct_call_sections(file); 4891 if (ret) 4892 goto out; 4893 } 4894 } 4895 4896 if (opts.mcount) { 4897 ret = create_mcount_loc_sections(file); 4898 if (ret) 4899 goto out; 4900 } 4901 4902 if (opts.prefix) { 4903 ret = create_prefix_symbols(file); 4904 if (ret) 4905 goto out; 4906 } 4907 4908 if (opts.ibt) { 4909 ret = create_ibt_endbr_seal_sections(file); 4910 if (ret) 4911 goto out; 4912 } 4913 4914 if (opts.noabs) 4915 warnings += check_abs_references(file); 4916 4917 if (opts.checksum) { 4918 ret = create_sym_checksum_section(file); 4919 if (ret) 4920 goto out; 4921 } 4922 4923 if (opts.orc && nr_insns) { 4924 ret = orc_create(file); 4925 if (ret) 4926 goto out; 4927 } 4928 4929 if (opts.stats) { 4930 printf("nr_insns_visited: %ld\n", nr_insns_visited); 4931 printf("nr_cfi: %ld\n", nr_cfi); 4932 printf("nr_cfi_reused: %ld\n", nr_cfi_reused); 4933 printf("nr_cfi_cache: %ld\n", nr_cfi_cache); 4934 } 4935 4936 out: 4937 if (!ret && !warnings) { 4938 free_insns(file); 4939 return 0; 4940 } 4941 4942 if (opts.werror && warnings) 4943 ret = 1; 4944 4945 if (opts.verbose) { 4946 if (opts.werror && warnings) 4947 WARN("%d warning(s) upgraded to errors", warnings); 4948 disas_ctx = disas_context_create(file); 4949 if (disas_ctx) { 4950 disas_warned_funcs(disas_ctx); 4951 disas_context_destroy(disas_ctx); 4952 } 4953 } 4954 4955 free_insns(file); 4956 4957 if (opts.backup && make_backup()) 4958 return 1; 4959 4960 return ret; 4961 } 4962