1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6 #define _GNU_SOURCE /* memmem() */ 7 #include <string.h> 8 #include <stdlib.h> 9 #include <inttypes.h> 10 #include <sys/mman.h> 11 12 #include <objtool/builtin.h> 13 #include <objtool/cfi.h> 14 #include <objtool/arch.h> 15 #include <objtool/check.h> 16 #include <objtool/special.h> 17 #include <objtool/warn.h> 18 #include <objtool/checksum.h> 19 #include <objtool/util.h> 20 21 #include <linux/objtool_types.h> 22 #include <linux/hashtable.h> 23 #include <linux/kernel.h> 24 #include <linux/static_call_types.h> 25 #include <linux/string.h> 26 27 struct alternative { 28 struct alternative *next; 29 struct instruction *insn; 30 }; 31 32 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 33 34 static struct cfi_init_state initial_func_cfi; 35 static struct cfi_state init_cfi; 36 static struct cfi_state func_cfi; 37 static struct cfi_state force_undefined_cfi; 38 39 struct instruction *find_insn(struct objtool_file *file, 40 struct section *sec, unsigned long offset) 41 { 42 struct instruction *insn; 43 44 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) { 45 if (insn->sec == sec && insn->offset == offset) 46 return insn; 47 } 48 49 return NULL; 50 } 51 52 struct instruction *next_insn_same_sec(struct objtool_file *file, 53 struct instruction *insn) 54 { 55 if (insn->idx == INSN_CHUNK_MAX) 56 return find_insn(file, insn->sec, insn->offset + insn->len); 57 58 insn++; 59 if (!insn->len) 60 return NULL; 61 62 return insn; 63 } 64 65 static struct instruction *next_insn_same_func(struct objtool_file *file, 66 struct instruction *insn) 67 { 68 struct instruction *next = next_insn_same_sec(file, insn); 69 struct symbol *func = insn_func(insn); 70 71 if (!func) 72 return NULL; 73 74 if (next && insn_func(next) == func) 75 return next; 76 77 /* Check if we're already in the subfunction: */ 78 if (func == func->cfunc) 79 return NULL; 80 81 /* Move to the subfunction: */ 82 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 83 } 84 85 static struct instruction *prev_insn_same_sec(struct objtool_file *file, 86 struct instruction *insn) 87 { 88 if (insn->idx == 0) { 89 if (insn->prev_len) 90 return find_insn(file, insn->sec, insn->offset - insn->prev_len); 91 return NULL; 92 } 93 94 return insn - 1; 95 } 96 97 static struct instruction *prev_insn_same_sym(struct objtool_file *file, 98 struct instruction *insn) 99 { 100 struct instruction *prev = prev_insn_same_sec(file, insn); 101 102 if (prev && insn_func(prev) == insn_func(insn)) 103 return prev; 104 105 return NULL; 106 } 107 108 #define for_each_insn(file, insn) \ 109 for (struct section *__sec, *__fake = (struct section *)1; \ 110 __fake; __fake = NULL) \ 111 for_each_sec(file->elf, __sec) \ 112 sec_for_each_insn(file, __sec, insn) 113 114 #define func_for_each_insn(file, func, insn) \ 115 for (insn = find_insn(file, func->sec, func->offset); \ 116 insn; \ 117 insn = next_insn_same_func(file, insn)) 118 119 #define sym_for_each_insn(file, sym, insn) \ 120 for (insn = find_insn(file, sym->sec, sym->offset); \ 121 insn && insn->offset < sym->offset + sym->len; \ 122 insn = next_insn_same_sec(file, insn)) 123 124 #define sym_for_each_insn_continue_reverse(file, sym, insn) \ 125 for (insn = prev_insn_same_sec(file, insn); \ 126 insn && insn->offset >= sym->offset; \ 127 insn = prev_insn_same_sec(file, insn)) 128 129 #define sec_for_each_insn_from(file, insn) \ 130 for (; insn; insn = next_insn_same_sec(file, insn)) 131 132 #define sec_for_each_insn_continue(file, insn) \ 133 for (insn = next_insn_same_sec(file, insn); insn; \ 134 insn = next_insn_same_sec(file, insn)) 135 136 static inline struct symbol *insn_call_dest(struct instruction *insn) 137 { 138 if (insn->type == INSN_JUMP_DYNAMIC || 139 insn->type == INSN_CALL_DYNAMIC) 140 return NULL; 141 142 return insn->_call_dest; 143 } 144 145 static inline struct reloc *insn_jump_table(struct instruction *insn) 146 { 147 if (insn->type == INSN_JUMP_DYNAMIC || 148 insn->type == INSN_CALL_DYNAMIC) 149 return insn->_jump_table; 150 151 return NULL; 152 } 153 154 static inline unsigned long insn_jump_table_size(struct instruction *insn) 155 { 156 if (insn->type == INSN_JUMP_DYNAMIC || 157 insn->type == INSN_CALL_DYNAMIC) 158 return insn->_jump_table_size; 159 160 return 0; 161 } 162 163 static bool is_jump_table_jump(struct instruction *insn) 164 { 165 struct alt_group *alt_group = insn->alt_group; 166 167 if (insn_jump_table(insn)) 168 return true; 169 170 /* Retpoline alternative for a jump table? */ 171 return alt_group && alt_group->orig_group && 172 insn_jump_table(alt_group->orig_group->first_insn); 173 } 174 175 static bool is_sibling_call(struct instruction *insn) 176 { 177 /* 178 * Assume only STT_FUNC calls have jump-tables. 179 */ 180 if (insn_func(insn)) { 181 /* An indirect jump is either a sibling call or a jump to a table. */ 182 if (insn->type == INSN_JUMP_DYNAMIC) 183 return !is_jump_table_jump(insn); 184 } 185 186 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */ 187 return (is_static_jump(insn) && insn_call_dest(insn)); 188 } 189 190 /* 191 * Checks if a function is a Rust "noreturn" one. 192 */ 193 static bool is_rust_noreturn(const struct symbol *func) 194 { 195 /* 196 * If it does not start with "_R", then it is not a Rust symbol. 197 */ 198 if (strncmp(func->name, "_R", 2)) 199 return false; 200 201 /* 202 * These are just heuristics -- we do not control the precise symbol 203 * name, due to the crate disambiguators (which depend on the compiler) 204 * as well as changes to the source code itself between versions (since 205 * these come from the Rust standard library). 206 */ 207 return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || 208 str_ends_with(func->name, "_4core6option13unwrap_failed") || 209 str_ends_with(func->name, "_4core6result13unwrap_failed") || 210 str_ends_with(func->name, "_4core9panicking5panic") || 211 str_ends_with(func->name, "_4core9panicking9panic_fmt") || 212 str_ends_with(func->name, "_4core9panicking14panic_explicit") || 213 str_ends_with(func->name, "_4core9panicking14panic_nounwind") || 214 str_ends_with(func->name, "_4core9panicking18panic_bounds_check") || 215 str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt") || 216 str_ends_with(func->name, "_4core9panicking19assert_failed_inner") || 217 str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference") || 218 str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") || 219 str_ends_with(func->name, "_7___rustc17rust_begin_unwind") || 220 strstr(func->name, "_4core9panicking13assert_failed") || 221 strstr(func->name, "_4core9panicking11panic_const24panic_const_") || 222 (strstr(func->name, "_4core5slice5index") && 223 strstr(func->name, "slice_") && 224 str_ends_with(func->name, "_fail")); 225 } 226 227 /* 228 * This checks to see if the given function is a "noreturn" function. 229 * 230 * For global functions which are outside the scope of this object file, we 231 * have to keep a manual list of them. 232 * 233 * For local functions, we have to detect them manually by simply looking for 234 * the lack of a return instruction. 235 */ 236 static bool __dead_end_function(struct objtool_file *file, struct symbol *func, 237 int recursion) 238 { 239 int i; 240 struct instruction *insn; 241 bool empty = true; 242 243 #define NORETURN(func) __stringify(func), 244 static const char * const global_noreturns[] = { 245 #include "noreturns.h" 246 }; 247 #undef NORETURN 248 249 if (!func) 250 return false; 251 252 if (!is_local_sym(func)) { 253 if (is_rust_noreturn(func)) 254 return true; 255 256 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) 257 if (!strcmp(func->name, global_noreturns[i])) 258 return true; 259 } 260 261 if (is_weak_sym(func)) 262 return false; 263 264 if (!func->len) 265 return false; 266 267 insn = find_insn(file, func->sec, func->offset); 268 if (!insn || !insn_func(insn)) 269 return false; 270 271 func_for_each_insn(file, func, insn) { 272 empty = false; 273 274 if (insn->type == INSN_RETURN) 275 return false; 276 } 277 278 if (empty) 279 return false; 280 281 /* 282 * A function can have a sibling call instead of a return. In that 283 * case, the function's dead-end status depends on whether the target 284 * of the sibling call returns. 285 */ 286 func_for_each_insn(file, func, insn) { 287 if (is_sibling_call(insn)) { 288 struct instruction *dest = insn->jump_dest; 289 290 if (!dest) 291 /* sibling call to another file */ 292 return false; 293 294 /* local sibling call */ 295 if (recursion == 5) { 296 /* 297 * Infinite recursion: two functions have 298 * sibling calls to each other. This is a very 299 * rare case. It means they aren't dead ends. 300 */ 301 return false; 302 } 303 304 return __dead_end_function(file, insn_func(dest), recursion+1); 305 } 306 } 307 308 return true; 309 } 310 311 static bool dead_end_function(struct objtool_file *file, struct symbol *func) 312 { 313 return __dead_end_function(file, func, 0); 314 } 315 316 static void init_cfi_state(struct cfi_state *cfi) 317 { 318 int i; 319 320 for (i = 0; i < CFI_NUM_REGS; i++) { 321 cfi->regs[i].base = CFI_UNDEFINED; 322 cfi->vals[i].base = CFI_UNDEFINED; 323 } 324 cfi->cfa.base = CFI_UNDEFINED; 325 cfi->drap_reg = CFI_UNDEFINED; 326 cfi->drap_offset = -1; 327 } 328 329 static void init_insn_state(struct objtool_file *file, struct insn_state *state, 330 struct section *sec) 331 { 332 memset(state, 0, sizeof(*state)); 333 init_cfi_state(&state->cfi); 334 335 if (opts.noinstr && sec) 336 state->noinstr = sec->noinstr; 337 } 338 339 static struct cfi_state *cfi_alloc(void) 340 { 341 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state)); 342 if (!cfi) { 343 ERROR_GLIBC("calloc"); 344 exit(1); 345 } 346 nr_cfi++; 347 return cfi; 348 } 349 350 static int cfi_bits; 351 static struct hlist_head *cfi_hash; 352 353 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2) 354 { 355 return memcmp((void *)cfi1 + sizeof(cfi1->hash), 356 (void *)cfi2 + sizeof(cfi2->hash), 357 sizeof(struct cfi_state) - sizeof(struct hlist_node)); 358 } 359 360 static inline u32 cfi_key(struct cfi_state *cfi) 361 { 362 return jhash((void *)cfi + sizeof(cfi->hash), 363 sizeof(*cfi) - sizeof(cfi->hash), 0); 364 } 365 366 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi) 367 { 368 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 369 struct cfi_state *obj; 370 371 hlist_for_each_entry(obj, head, hash) { 372 if (!cficmp(cfi, obj)) { 373 nr_cfi_cache++; 374 return obj; 375 } 376 } 377 378 obj = cfi_alloc(); 379 *obj = *cfi; 380 hlist_add_head(&obj->hash, head); 381 382 return obj; 383 } 384 385 static void cfi_hash_add(struct cfi_state *cfi) 386 { 387 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 388 389 hlist_add_head(&cfi->hash, head); 390 } 391 392 static void *cfi_hash_alloc(unsigned long size) 393 { 394 cfi_bits = max(10, ilog2(size)); 395 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits, 396 PROT_READ|PROT_WRITE, 397 MAP_PRIVATE|MAP_ANON, -1, 0); 398 if (cfi_hash == (void *)-1L) { 399 ERROR_GLIBC("mmap fail cfi_hash"); 400 cfi_hash = NULL; 401 } else if (opts.stats) { 402 printf("cfi_bits: %d\n", cfi_bits); 403 } 404 405 return cfi_hash; 406 } 407 408 static unsigned long nr_insns; 409 static unsigned long nr_insns_visited; 410 411 /* 412 * Call the arch-specific instruction decoder for all the instructions and add 413 * them to the global instruction list. 414 */ 415 static int decode_instructions(struct objtool_file *file) 416 { 417 struct section *sec; 418 struct symbol *func; 419 unsigned long offset; 420 struct instruction *insn; 421 422 for_each_sec(file->elf, sec) { 423 struct instruction *insns = NULL; 424 u8 prev_len = 0; 425 u8 idx = 0; 426 427 if (!is_text_sec(sec)) 428 continue; 429 430 if (strcmp(sec->name, ".altinstr_replacement") && 431 strcmp(sec->name, ".altinstr_aux") && 432 strncmp(sec->name, ".discard.", 9)) 433 sec->text = true; 434 435 if (!strcmp(sec->name, ".noinstr.text") || 436 !strcmp(sec->name, ".entry.text") || 437 !strcmp(sec->name, ".cpuidle.text") || 438 !strncmp(sec->name, ".text..__x86.", 13)) 439 sec->noinstr = true; 440 441 /* 442 * .init.text code is ran before userspace and thus doesn't 443 * strictly need retpolines, except for modules which are 444 * loaded late, they very much do need retpoline in their 445 * .init.text 446 */ 447 if (!strcmp(sec->name, ".init.text") && !opts.module) 448 sec->init = true; 449 450 for (offset = 0; offset < sec_size(sec); offset += insn->len) { 451 if (!insns || idx == INSN_CHUNK_MAX) { 452 insns = calloc(INSN_CHUNK_SIZE, sizeof(*insn)); 453 if (!insns) { 454 ERROR_GLIBC("calloc"); 455 return -1; 456 } 457 idx = 0; 458 } else { 459 idx++; 460 } 461 insn = &insns[idx]; 462 insn->idx = idx; 463 464 INIT_LIST_HEAD(&insn->call_node); 465 insn->sec = sec; 466 insn->offset = offset; 467 insn->prev_len = prev_len; 468 469 if (arch_decode_instruction(file, sec, offset, sec_size(sec) - offset, insn)) 470 return -1; 471 472 prev_len = insn->len; 473 474 /* 475 * By default, "ud2" is a dead end unless otherwise 476 * annotated, because GCC 7 inserts it for certain 477 * divide-by-zero cases. 478 */ 479 if (insn->type == INSN_BUG) 480 insn->dead_end = true; 481 482 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset)); 483 nr_insns++; 484 } 485 486 sec_for_each_sym(sec, func) { 487 if (!is_notype_sym(func) && !is_func_sym(func)) 488 continue; 489 490 if (func->offset == sec_size(sec)) { 491 /* Heuristic: likely an "end" symbol */ 492 if (is_notype_sym(func)) 493 continue; 494 ERROR("%s(): STT_FUNC at end of section", func->name); 495 return -1; 496 } 497 498 if (func->embedded_insn || func->alias != func) 499 continue; 500 501 if (!find_insn(file, sec, func->offset)) { 502 ERROR("%s(): can't find starting instruction", func->name); 503 return -1; 504 } 505 506 sym_for_each_insn(file, func, insn) { 507 insn->sym = func; 508 if (is_func_sym(func) && 509 insn->type == INSN_ENDBR && 510 list_empty(&insn->call_node)) { 511 if (insn->offset == func->offset) { 512 list_add_tail(&insn->call_node, &file->endbr_list); 513 file->nr_endbr++; 514 } else { 515 file->nr_endbr_int++; 516 } 517 } 518 } 519 } 520 } 521 522 if (opts.stats) 523 printf("nr_insns: %lu\n", nr_insns); 524 525 return 0; 526 } 527 528 /* 529 * Read the pv_ops[] .data table to find the static initialized values. 530 */ 531 static int add_pv_ops(struct objtool_file *file, const char *symname) 532 { 533 struct symbol *sym, *func; 534 unsigned long off, end; 535 struct reloc *reloc; 536 int idx; 537 538 sym = find_symbol_by_name(file->elf, symname); 539 if (!sym) 540 return 0; 541 542 off = sym->offset; 543 end = off + sym->len; 544 for (;;) { 545 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 546 if (!reloc) 547 break; 548 549 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long); 550 551 func = reloc->sym; 552 if (is_sec_sym(func)) 553 func = find_symbol_by_offset(reloc->sym->sec, 554 reloc_addend(reloc)); 555 if (!func) { 556 ERROR_FUNC(reloc->sym->sec, reloc_addend(reloc), 557 "can't find func at %s[%d]", symname, idx); 558 return -1; 559 } 560 561 if (objtool_pv_add(file, idx, func)) 562 return -1; 563 564 off = reloc_offset(reloc) + 1; 565 if (off > end) 566 break; 567 } 568 569 return 0; 570 } 571 572 /* 573 * Allocate and initialize file->pv_ops[]. 574 */ 575 static int init_pv_ops(struct objtool_file *file) 576 { 577 static const char *pv_ops_tables[] = { 578 "pv_ops", 579 "xen_cpu_ops", 580 "xen_irq_ops", 581 "xen_mmu_ops", 582 NULL, 583 }; 584 const char *pv_ops; 585 struct symbol *sym; 586 int idx, nr; 587 588 if (!opts.noinstr) 589 return 0; 590 591 file->pv_ops = NULL; 592 593 sym = find_symbol_by_name(file->elf, "pv_ops"); 594 if (!sym) 595 return 0; 596 597 nr = sym->len / sizeof(unsigned long); 598 file->pv_ops = calloc(nr, sizeof(struct pv_state)); 599 if (!file->pv_ops) { 600 ERROR_GLIBC("calloc"); 601 return -1; 602 } 603 604 for (idx = 0; idx < nr; idx++) 605 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 606 607 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) { 608 if (add_pv_ops(file, pv_ops)) 609 return -1; 610 } 611 612 return 0; 613 } 614 615 static bool is_livepatch_module(struct objtool_file *file) 616 { 617 struct section *sec; 618 619 if (!opts.module) 620 return false; 621 622 sec = find_section_by_name(file->elf, ".modinfo"); 623 if (!sec) 624 return false; 625 626 return memmem(sec->data->d_buf, sec_size(sec), "\0livepatch=Y", 12); 627 } 628 629 static int create_static_call_sections(struct objtool_file *file) 630 { 631 struct static_call_site *site; 632 struct section *sec; 633 struct instruction *insn; 634 struct symbol *key_sym; 635 char *key_name, *tmp; 636 int idx; 637 638 sec = find_section_by_name(file->elf, ".static_call_sites"); 639 if (sec) { 640 /* 641 * Livepatch modules may have already extracted the static call 642 * site entries to take advantage of vmlinux static call 643 * privileges. 644 */ 645 if (!file->klp) 646 WARN("file already has .static_call_sites section, skipping"); 647 648 return 0; 649 } 650 651 if (list_empty(&file->static_call_list)) 652 return 0; 653 654 idx = 0; 655 list_for_each_entry(insn, &file->static_call_list, call_node) 656 idx++; 657 658 sec = elf_create_section_pair(file->elf, ".static_call_sites", 659 sizeof(*site), idx, idx * 2); 660 if (!sec) 661 return -1; 662 663 /* Allow modules to modify the low bits of static_call_site::key */ 664 sec->sh.sh_flags |= SHF_WRITE; 665 666 idx = 0; 667 list_for_each_entry(insn, &file->static_call_list, call_node) { 668 669 /* populate reloc for 'addr' */ 670 if (!elf_init_reloc_text_sym(file->elf, sec, 671 idx * sizeof(*site), idx * 2, 672 insn->sec, insn->offset)) 673 return -1; 674 675 /* find key symbol */ 676 key_name = strdup(insn_call_dest(insn)->name); 677 if (!key_name) { 678 ERROR_GLIBC("strdup"); 679 return -1; 680 } 681 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR, 682 STATIC_CALL_TRAMP_PREFIX_LEN)) { 683 ERROR("static_call: trampoline name malformed: %s", key_name); 684 return -1; 685 } 686 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN; 687 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN); 688 689 key_sym = find_symbol_by_name(file->elf, tmp); 690 if (!key_sym) { 691 if (!opts.module || file->klp) { 692 ERROR("static_call: can't find static_call_key symbol: %s", tmp); 693 return -1; 694 } 695 696 /* 697 * For modules(), the key might not be exported, which 698 * means the module can make static calls but isn't 699 * allowed to change them. 700 * 701 * In that case we temporarily set the key to be the 702 * trampoline address. This is fixed up in 703 * static_call_add_module(). 704 */ 705 key_sym = insn_call_dest(insn); 706 } 707 708 /* populate reloc for 'key' */ 709 if (!elf_init_reloc_data_sym(file->elf, sec, 710 idx * sizeof(*site) + 4, 711 (idx * 2) + 1, key_sym, 712 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL)) 713 return -1; 714 715 idx++; 716 } 717 718 return 0; 719 } 720 721 static int create_retpoline_sites_sections(struct objtool_file *file) 722 { 723 struct instruction *insn; 724 struct section *sec; 725 int idx; 726 727 sec = find_section_by_name(file->elf, ".retpoline_sites"); 728 if (sec) { 729 WARN("file already has .retpoline_sites, skipping"); 730 return 0; 731 } 732 733 idx = 0; 734 list_for_each_entry(insn, &file->retpoline_call_list, call_node) 735 idx++; 736 737 if (!idx) 738 return 0; 739 740 sec = elf_create_section_pair(file->elf, ".retpoline_sites", 741 sizeof(int), idx, idx); 742 if (!sec) 743 return -1; 744 745 idx = 0; 746 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 747 748 if (!elf_init_reloc_text_sym(file->elf, sec, 749 idx * sizeof(int), idx, 750 insn->sec, insn->offset)) 751 return -1; 752 753 idx++; 754 } 755 756 return 0; 757 } 758 759 static int create_return_sites_sections(struct objtool_file *file) 760 { 761 struct instruction *insn; 762 struct section *sec; 763 int idx; 764 765 sec = find_section_by_name(file->elf, ".return_sites"); 766 if (sec) { 767 WARN("file already has .return_sites, skipping"); 768 return 0; 769 } 770 771 idx = 0; 772 list_for_each_entry(insn, &file->return_thunk_list, call_node) 773 idx++; 774 775 if (!idx) 776 return 0; 777 778 sec = elf_create_section_pair(file->elf, ".return_sites", 779 sizeof(int), idx, idx); 780 if (!sec) 781 return -1; 782 783 idx = 0; 784 list_for_each_entry(insn, &file->return_thunk_list, call_node) { 785 786 if (!elf_init_reloc_text_sym(file->elf, sec, 787 idx * sizeof(int), idx, 788 insn->sec, insn->offset)) 789 return -1; 790 791 idx++; 792 } 793 794 return 0; 795 } 796 797 static int create_ibt_endbr_seal_sections(struct objtool_file *file) 798 { 799 struct instruction *insn; 800 struct section *sec; 801 int idx; 802 803 sec = find_section_by_name(file->elf, ".ibt_endbr_seal"); 804 if (sec) { 805 WARN("file already has .ibt_endbr_seal, skipping"); 806 return 0; 807 } 808 809 idx = 0; 810 list_for_each_entry(insn, &file->endbr_list, call_node) 811 idx++; 812 813 if (opts.stats) { 814 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); 815 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); 816 printf("ibt: superfluous ENDBR: %d\n", idx); 817 } 818 819 if (!idx) 820 return 0; 821 822 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal", 823 sizeof(int), idx, idx); 824 if (!sec) 825 return -1; 826 827 idx = 0; 828 list_for_each_entry(insn, &file->endbr_list, call_node) { 829 830 int *site = (int *)sec->data->d_buf + idx; 831 struct symbol *sym = insn->sym; 832 *site = 0; 833 834 if (opts.module && sym && is_func_sym(sym) && 835 insn->offset == sym->offset && 836 (!strcmp(sym->name, "init_module") || 837 !strcmp(sym->name, "cleanup_module"))) { 838 ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead", 839 sym->name); 840 return -1; 841 } 842 843 if (!elf_init_reloc_text_sym(file->elf, sec, 844 idx * sizeof(int), idx, 845 insn->sec, insn->offset)) 846 return -1; 847 848 idx++; 849 } 850 851 return 0; 852 } 853 854 static int create_cfi_sections(struct objtool_file *file) 855 { 856 struct section *sec; 857 struct symbol *sym; 858 int idx; 859 860 sec = find_section_by_name(file->elf, ".cfi_sites"); 861 if (sec) { 862 WARN("file already has .cfi_sites section, skipping"); 863 return 0; 864 } 865 866 idx = 0; 867 for_each_sym(file->elf, sym) { 868 if (!is_func_sym(sym)) 869 continue; 870 871 if (strncmp(sym->name, "__cfi_", 6)) 872 continue; 873 874 idx++; 875 } 876 877 sec = elf_create_section_pair(file->elf, ".cfi_sites", 878 sizeof(unsigned int), idx, idx); 879 if (!sec) 880 return -1; 881 882 idx = 0; 883 for_each_sym(file->elf, sym) { 884 if (!is_func_sym(sym)) 885 continue; 886 887 if (strncmp(sym->name, "__cfi_", 6)) 888 continue; 889 890 if (!elf_init_reloc_text_sym(file->elf, sec, 891 idx * sizeof(unsigned int), idx, 892 sym->sec, sym->offset)) 893 return -1; 894 895 idx++; 896 } 897 898 return 0; 899 } 900 901 static int create_mcount_loc_sections(struct objtool_file *file) 902 { 903 size_t addr_size = elf_addr_size(file->elf); 904 struct instruction *insn; 905 struct section *sec; 906 int idx; 907 908 sec = find_section_by_name(file->elf, "__mcount_loc"); 909 if (sec) { 910 /* 911 * Livepatch modules have already extracted their __mcount_loc 912 * entries to cover the !CONFIG_FTRACE_MCOUNT_USE_OBJTOOL case. 913 */ 914 if (!file->klp) 915 WARN("file already has __mcount_loc section, skipping"); 916 917 return 0; 918 } 919 920 if (list_empty(&file->mcount_loc_list)) 921 return 0; 922 923 idx = 0; 924 list_for_each_entry(insn, &file->mcount_loc_list, call_node) 925 idx++; 926 927 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size, 928 idx, idx); 929 if (!sec) 930 return -1; 931 932 sec->sh.sh_addralign = addr_size; 933 934 idx = 0; 935 list_for_each_entry(insn, &file->mcount_loc_list, call_node) { 936 937 struct reloc *reloc; 938 939 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx, 940 insn->sec, insn->offset); 941 if (!reloc) 942 return -1; 943 944 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32); 945 946 idx++; 947 } 948 949 return 0; 950 } 951 952 static int create_direct_call_sections(struct objtool_file *file) 953 { 954 struct instruction *insn; 955 struct section *sec; 956 int idx; 957 958 sec = find_section_by_name(file->elf, ".call_sites"); 959 if (sec) { 960 WARN("file already has .call_sites section, skipping"); 961 return 0; 962 } 963 964 if (list_empty(&file->call_list)) 965 return 0; 966 967 idx = 0; 968 list_for_each_entry(insn, &file->call_list, call_node) 969 idx++; 970 971 sec = elf_create_section_pair(file->elf, ".call_sites", 972 sizeof(unsigned int), idx, idx); 973 if (!sec) 974 return -1; 975 976 idx = 0; 977 list_for_each_entry(insn, &file->call_list, call_node) { 978 979 if (!elf_init_reloc_text_sym(file->elf, sec, 980 idx * sizeof(unsigned int), idx, 981 insn->sec, insn->offset)) 982 return -1; 983 984 idx++; 985 } 986 987 return 0; 988 } 989 990 #ifdef BUILD_KLP 991 static int create_sym_checksum_section(struct objtool_file *file) 992 { 993 struct section *sec; 994 struct symbol *sym; 995 unsigned int idx = 0; 996 struct sym_checksum *checksum; 997 size_t entsize = sizeof(struct sym_checksum); 998 999 sec = find_section_by_name(file->elf, ".discard.sym_checksum"); 1000 if (sec) { 1001 if (!opts.dryrun) 1002 WARN("file already has .discard.sym_checksum section, skipping"); 1003 1004 return 0; 1005 } 1006 1007 for_each_sym(file->elf, sym) 1008 if (sym->csum.checksum) 1009 idx++; 1010 1011 if (!idx) 1012 return 0; 1013 1014 sec = elf_create_section_pair(file->elf, ".discard.sym_checksum", entsize, 1015 idx, idx); 1016 if (!sec) 1017 return -1; 1018 1019 idx = 0; 1020 for_each_sym(file->elf, sym) { 1021 if (!sym->csum.checksum) 1022 continue; 1023 1024 if (!elf_init_reloc(file->elf, sec->rsec, idx, idx * entsize, 1025 sym, 0, R_TEXT64)) 1026 return -1; 1027 1028 checksum = (struct sym_checksum *)sec->data->d_buf + idx; 1029 checksum->addr = 0; /* reloc */ 1030 checksum->checksum = sym->csum.checksum; 1031 1032 mark_sec_changed(file->elf, sec, true); 1033 1034 idx++; 1035 } 1036 1037 return 0; 1038 } 1039 #else 1040 static int create_sym_checksum_section(struct objtool_file *file) { return -EINVAL; } 1041 #endif 1042 1043 /* 1044 * Warnings shouldn't be reported for ignored functions. 1045 */ 1046 static int add_ignores(struct objtool_file *file) 1047 { 1048 struct section *rsec; 1049 struct symbol *func; 1050 struct reloc *reloc; 1051 1052 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard"); 1053 if (!rsec) 1054 return 0; 1055 1056 for_each_reloc(rsec, reloc) { 1057 switch (reloc->sym->type) { 1058 case STT_FUNC: 1059 func = reloc->sym; 1060 break; 1061 1062 case STT_SECTION: 1063 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc)); 1064 if (!func) 1065 continue; 1066 break; 1067 1068 default: 1069 ERROR("unexpected relocation symbol type in %s: %d", 1070 rsec->name, reloc->sym->type); 1071 return -1; 1072 } 1073 1074 func->ignore = true; 1075 if (func->cfunc) 1076 func->cfunc->ignore = true; 1077 } 1078 1079 return 0; 1080 } 1081 1082 /* 1083 * This is a whitelist of functions that is allowed to be called with AC set. 1084 * The list is meant to be minimal and only contains compiler instrumentation 1085 * ABI and a few functions used to implement *_{to,from}_user() functions. 1086 * 1087 * These functions must not directly change AC, but may PUSHF/POPF. 1088 */ 1089 static const char *uaccess_safe_builtin[] = { 1090 /* KASAN */ 1091 "kasan_report", 1092 "kasan_check_range", 1093 /* KASAN out-of-line */ 1094 "__asan_loadN_noabort", 1095 "__asan_load1_noabort", 1096 "__asan_load2_noabort", 1097 "__asan_load4_noabort", 1098 "__asan_load8_noabort", 1099 "__asan_load16_noabort", 1100 "__asan_storeN_noabort", 1101 "__asan_store1_noabort", 1102 "__asan_store2_noabort", 1103 "__asan_store4_noabort", 1104 "__asan_store8_noabort", 1105 "__asan_store16_noabort", 1106 "__kasan_check_read", 1107 "__kasan_check_write", 1108 /* KASAN in-line */ 1109 "__asan_report_load_n_noabort", 1110 "__asan_report_load1_noabort", 1111 "__asan_report_load2_noabort", 1112 "__asan_report_load4_noabort", 1113 "__asan_report_load8_noabort", 1114 "__asan_report_load16_noabort", 1115 "__asan_report_store_n_noabort", 1116 "__asan_report_store1_noabort", 1117 "__asan_report_store2_noabort", 1118 "__asan_report_store4_noabort", 1119 "__asan_report_store8_noabort", 1120 "__asan_report_store16_noabort", 1121 /* KCSAN */ 1122 "__kcsan_check_access", 1123 "__kcsan_mb", 1124 "__kcsan_wmb", 1125 "__kcsan_rmb", 1126 "__kcsan_release", 1127 "kcsan_found_watchpoint", 1128 "kcsan_setup_watchpoint", 1129 "kcsan_check_scoped_accesses", 1130 "kcsan_disable_current", 1131 "kcsan_enable_current_nowarn", 1132 /* KCSAN/TSAN */ 1133 "__tsan_func_entry", 1134 "__tsan_func_exit", 1135 "__tsan_read_range", 1136 "__tsan_write_range", 1137 "__tsan_read1", 1138 "__tsan_read2", 1139 "__tsan_read4", 1140 "__tsan_read8", 1141 "__tsan_read16", 1142 "__tsan_write1", 1143 "__tsan_write2", 1144 "__tsan_write4", 1145 "__tsan_write8", 1146 "__tsan_write16", 1147 "__tsan_read_write1", 1148 "__tsan_read_write2", 1149 "__tsan_read_write4", 1150 "__tsan_read_write8", 1151 "__tsan_read_write16", 1152 "__tsan_volatile_read1", 1153 "__tsan_volatile_read2", 1154 "__tsan_volatile_read4", 1155 "__tsan_volatile_read8", 1156 "__tsan_volatile_read16", 1157 "__tsan_volatile_write1", 1158 "__tsan_volatile_write2", 1159 "__tsan_volatile_write4", 1160 "__tsan_volatile_write8", 1161 "__tsan_volatile_write16", 1162 "__tsan_atomic8_load", 1163 "__tsan_atomic16_load", 1164 "__tsan_atomic32_load", 1165 "__tsan_atomic64_load", 1166 "__tsan_atomic8_store", 1167 "__tsan_atomic16_store", 1168 "__tsan_atomic32_store", 1169 "__tsan_atomic64_store", 1170 "__tsan_atomic8_exchange", 1171 "__tsan_atomic16_exchange", 1172 "__tsan_atomic32_exchange", 1173 "__tsan_atomic64_exchange", 1174 "__tsan_atomic8_fetch_add", 1175 "__tsan_atomic16_fetch_add", 1176 "__tsan_atomic32_fetch_add", 1177 "__tsan_atomic64_fetch_add", 1178 "__tsan_atomic8_fetch_sub", 1179 "__tsan_atomic16_fetch_sub", 1180 "__tsan_atomic32_fetch_sub", 1181 "__tsan_atomic64_fetch_sub", 1182 "__tsan_atomic8_fetch_and", 1183 "__tsan_atomic16_fetch_and", 1184 "__tsan_atomic32_fetch_and", 1185 "__tsan_atomic64_fetch_and", 1186 "__tsan_atomic8_fetch_or", 1187 "__tsan_atomic16_fetch_or", 1188 "__tsan_atomic32_fetch_or", 1189 "__tsan_atomic64_fetch_or", 1190 "__tsan_atomic8_fetch_xor", 1191 "__tsan_atomic16_fetch_xor", 1192 "__tsan_atomic32_fetch_xor", 1193 "__tsan_atomic64_fetch_xor", 1194 "__tsan_atomic8_fetch_nand", 1195 "__tsan_atomic16_fetch_nand", 1196 "__tsan_atomic32_fetch_nand", 1197 "__tsan_atomic64_fetch_nand", 1198 "__tsan_atomic8_compare_exchange_strong", 1199 "__tsan_atomic16_compare_exchange_strong", 1200 "__tsan_atomic32_compare_exchange_strong", 1201 "__tsan_atomic64_compare_exchange_strong", 1202 "__tsan_atomic8_compare_exchange_weak", 1203 "__tsan_atomic16_compare_exchange_weak", 1204 "__tsan_atomic32_compare_exchange_weak", 1205 "__tsan_atomic64_compare_exchange_weak", 1206 "__tsan_atomic8_compare_exchange_val", 1207 "__tsan_atomic16_compare_exchange_val", 1208 "__tsan_atomic32_compare_exchange_val", 1209 "__tsan_atomic64_compare_exchange_val", 1210 "__tsan_atomic_thread_fence", 1211 "__tsan_atomic_signal_fence", 1212 "__tsan_unaligned_read16", 1213 "__tsan_unaligned_write16", 1214 /* KCOV */ 1215 "write_comp_data", 1216 "check_kcov_mode", 1217 "__sanitizer_cov_trace_pc", 1218 "__sanitizer_cov_trace_const_cmp1", 1219 "__sanitizer_cov_trace_const_cmp2", 1220 "__sanitizer_cov_trace_const_cmp4", 1221 "__sanitizer_cov_trace_const_cmp8", 1222 "__sanitizer_cov_trace_cmp1", 1223 "__sanitizer_cov_trace_cmp2", 1224 "__sanitizer_cov_trace_cmp4", 1225 "__sanitizer_cov_trace_cmp8", 1226 "__sanitizer_cov_trace_switch", 1227 /* KMSAN */ 1228 "kmsan_copy_to_user", 1229 "kmsan_disable_current", 1230 "kmsan_enable_current", 1231 "kmsan_report", 1232 "kmsan_unpoison_entry_regs", 1233 "kmsan_unpoison_memory", 1234 "__msan_chain_origin", 1235 "__msan_get_context_state", 1236 "__msan_instrument_asm_store", 1237 "__msan_metadata_ptr_for_load_1", 1238 "__msan_metadata_ptr_for_load_2", 1239 "__msan_metadata_ptr_for_load_4", 1240 "__msan_metadata_ptr_for_load_8", 1241 "__msan_metadata_ptr_for_load_n", 1242 "__msan_metadata_ptr_for_store_1", 1243 "__msan_metadata_ptr_for_store_2", 1244 "__msan_metadata_ptr_for_store_4", 1245 "__msan_metadata_ptr_for_store_8", 1246 "__msan_metadata_ptr_for_store_n", 1247 "__msan_poison_alloca", 1248 "__msan_warning", 1249 /* UBSAN */ 1250 "ubsan_type_mismatch_common", 1251 "__ubsan_handle_type_mismatch", 1252 "__ubsan_handle_type_mismatch_v1", 1253 "__ubsan_handle_shift_out_of_bounds", 1254 "__ubsan_handle_load_invalid_value", 1255 /* KSTACK_ERASE */ 1256 "__sanitizer_cov_stack_depth", 1257 /* TRACE_BRANCH_PROFILING */ 1258 "ftrace_likely_update", 1259 /* STACKPROTECTOR */ 1260 "__stack_chk_fail", 1261 /* misc */ 1262 "csum_partial_copy_generic", 1263 "copy_mc_fragile", 1264 "copy_mc_fragile_handle_tail", 1265 "copy_mc_enhanced_fast_string", 1266 "rep_stos_alternative", 1267 "rep_movs_alternative", 1268 "__copy_user_nocache", 1269 NULL 1270 }; 1271 1272 static void add_uaccess_safe(struct objtool_file *file) 1273 { 1274 struct symbol *func; 1275 const char **name; 1276 1277 if (!opts.uaccess) 1278 return; 1279 1280 for (name = uaccess_safe_builtin; *name; name++) { 1281 func = find_symbol_by_name(file->elf, *name); 1282 if (!func) 1283 continue; 1284 1285 func->uaccess_safe = true; 1286 } 1287 } 1288 1289 /* 1290 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol 1291 * will be added to the .retpoline_sites section. 1292 */ 1293 __weak bool arch_is_retpoline(struct symbol *sym) 1294 { 1295 return false; 1296 } 1297 1298 /* 1299 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol 1300 * will be added to the .return_sites section. 1301 */ 1302 __weak bool arch_is_rethunk(struct symbol *sym) 1303 { 1304 return false; 1305 } 1306 1307 /* 1308 * Symbols that are embedded inside other instructions, because sometimes crazy 1309 * code exists. These are mostly ignored for validation purposes. 1310 */ 1311 __weak bool arch_is_embedded_insn(struct symbol *sym) 1312 { 1313 return false; 1314 } 1315 1316 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) 1317 { 1318 struct reloc *reloc; 1319 1320 if (insn->no_reloc) 1321 return NULL; 1322 1323 if (!file) 1324 return NULL; 1325 1326 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 1327 insn->offset, insn->len); 1328 if (!reloc) { 1329 insn->no_reloc = 1; 1330 return NULL; 1331 } 1332 1333 return reloc; 1334 } 1335 1336 static void remove_insn_ops(struct instruction *insn) 1337 { 1338 struct stack_op *op, *next; 1339 1340 for (op = insn->stack_ops; op; op = next) { 1341 next = op->next; 1342 free(op); 1343 } 1344 insn->stack_ops = NULL; 1345 } 1346 1347 static int annotate_call_site(struct objtool_file *file, 1348 struct instruction *insn, bool sibling) 1349 { 1350 struct reloc *reloc = insn_reloc(file, insn); 1351 struct symbol *sym = insn_call_dest(insn); 1352 1353 if (!sym) 1354 sym = reloc->sym; 1355 1356 if (sym->static_call_tramp) { 1357 list_add_tail(&insn->call_node, &file->static_call_list); 1358 return 0; 1359 } 1360 1361 if (sym->retpoline_thunk) { 1362 list_add_tail(&insn->call_node, &file->retpoline_call_list); 1363 return 0; 1364 } 1365 1366 /* 1367 * Many compilers cannot disable KCOV or sanitizer calls with a function 1368 * attribute so they need a little help, NOP out any such calls from 1369 * noinstr text. 1370 */ 1371 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) { 1372 if (reloc) 1373 set_reloc_type(file->elf, reloc, R_NONE); 1374 1375 if (elf_write_insn(file->elf, insn->sec, 1376 insn->offset, insn->len, 1377 sibling ? arch_ret_insn(insn->len) 1378 : arch_nop_insn(insn->len))) { 1379 return -1; 1380 } 1381 1382 insn->type = sibling ? INSN_RETURN : INSN_NOP; 1383 1384 if (sibling) { 1385 /* 1386 * We've replaced the tail-call JMP insn by two new 1387 * insn: RET; INT3, except we only have a single struct 1388 * insn here. Mark it retpoline_safe to avoid the SLS 1389 * warning, instead of adding another insn. 1390 */ 1391 insn->retpoline_safe = true; 1392 } 1393 1394 return 0; 1395 } 1396 1397 if (opts.mcount && sym->fentry) { 1398 if (sibling) 1399 WARN_INSN(insn, "tail call to __fentry__ !?!?"); 1400 if (opts.mnop) { 1401 if (reloc) 1402 set_reloc_type(file->elf, reloc, R_NONE); 1403 1404 if (elf_write_insn(file->elf, insn->sec, 1405 insn->offset, insn->len, 1406 arch_nop_insn(insn->len))) { 1407 return -1; 1408 } 1409 1410 insn->type = INSN_NOP; 1411 } 1412 1413 list_add_tail(&insn->call_node, &file->mcount_loc_list); 1414 return 0; 1415 } 1416 1417 if (insn->type == INSN_CALL && !insn->sec->init && 1418 !insn->_call_dest->embedded_insn) 1419 list_add_tail(&insn->call_node, &file->call_list); 1420 1421 if (!sibling && dead_end_function(file, sym)) 1422 insn->dead_end = true; 1423 1424 return 0; 1425 } 1426 1427 static int add_call_dest(struct objtool_file *file, struct instruction *insn, 1428 struct symbol *dest, bool sibling) 1429 { 1430 insn->_call_dest = dest; 1431 if (!dest) 1432 return 0; 1433 1434 /* 1435 * Whatever stack impact regular CALLs have, should be undone 1436 * by the RETURN of the called function. 1437 * 1438 * Annotated intra-function calls retain the stack_ops but 1439 * are converted to JUMP, see read_intra_function_calls(). 1440 */ 1441 remove_insn_ops(insn); 1442 1443 return annotate_call_site(file, insn, sibling); 1444 } 1445 1446 static int add_retpoline_call(struct objtool_file *file, struct instruction *insn) 1447 { 1448 /* 1449 * Retpoline calls/jumps are really dynamic calls/jumps in disguise, 1450 * so convert them accordingly. 1451 */ 1452 switch (insn->type) { 1453 case INSN_CALL: 1454 insn->type = INSN_CALL_DYNAMIC; 1455 break; 1456 case INSN_JUMP_UNCONDITIONAL: 1457 insn->type = INSN_JUMP_DYNAMIC; 1458 break; 1459 case INSN_JUMP_CONDITIONAL: 1460 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL; 1461 break; 1462 default: 1463 return 0; 1464 } 1465 1466 insn->retpoline_safe = true; 1467 1468 /* 1469 * Whatever stack impact regular CALLs have, should be undone 1470 * by the RETURN of the called function. 1471 * 1472 * Annotated intra-function calls retain the stack_ops but 1473 * are converted to JUMP, see read_intra_function_calls(). 1474 */ 1475 remove_insn_ops(insn); 1476 1477 return annotate_call_site(file, insn, false); 1478 } 1479 1480 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) 1481 { 1482 /* 1483 * Return thunk tail calls are really just returns in disguise, 1484 * so convert them accordingly. 1485 */ 1486 insn->type = INSN_RETURN; 1487 insn->retpoline_safe = true; 1488 1489 if (add) 1490 list_add_tail(&insn->call_node, &file->return_thunk_list); 1491 } 1492 1493 static bool is_first_func_insn(struct objtool_file *file, 1494 struct instruction *insn) 1495 { 1496 struct symbol *func = insn_func(insn); 1497 1498 if (!func) 1499 return false; 1500 1501 if (insn->offset == func->offset) 1502 return true; 1503 1504 /* Allow direct CALL/JMP past ENDBR */ 1505 if (opts.ibt) { 1506 struct instruction *prev = prev_insn_same_sym(file, insn); 1507 1508 if (prev && prev->type == INSN_ENDBR && 1509 insn->offset == func->offset + prev->len) 1510 return true; 1511 } 1512 1513 return false; 1514 } 1515 1516 /* 1517 * Find the destination instructions for all jumps. 1518 */ 1519 static int add_jump_destinations(struct objtool_file *file) 1520 { 1521 struct instruction *insn; 1522 struct reloc *reloc; 1523 1524 for_each_insn(file, insn) { 1525 struct symbol *func = insn_func(insn); 1526 struct instruction *dest_insn; 1527 struct section *dest_sec; 1528 struct symbol *dest_sym; 1529 unsigned long dest_off; 1530 1531 if (!is_static_jump(insn)) 1532 continue; 1533 1534 if (insn->jump_dest) { 1535 /* 1536 * handle_group_alt() may have previously set 1537 * 'jump_dest' for some alternatives. 1538 */ 1539 continue; 1540 } 1541 1542 reloc = insn_reloc(file, insn); 1543 if (!reloc) { 1544 dest_sec = insn->sec; 1545 dest_off = arch_jump_destination(insn); 1546 dest_sym = dest_sec->sym; 1547 } else { 1548 dest_sym = reloc->sym; 1549 if (is_undef_sym(dest_sym)) { 1550 if (dest_sym->retpoline_thunk) { 1551 if (add_retpoline_call(file, insn)) 1552 return -1; 1553 continue; 1554 } 1555 1556 if (dest_sym->return_thunk) { 1557 add_return_call(file, insn, true); 1558 continue; 1559 } 1560 1561 /* External symbol */ 1562 if (func) { 1563 /* External sibling call */ 1564 if (add_call_dest(file, insn, dest_sym, true)) 1565 return -1; 1566 continue; 1567 } 1568 1569 /* Non-func asm code jumping to external symbol */ 1570 continue; 1571 } 1572 1573 dest_sec = dest_sym->sec; 1574 dest_off = dest_sym->offset + arch_insn_adjusted_addend(insn, reloc); 1575 } 1576 1577 dest_insn = find_insn(file, dest_sec, dest_off); 1578 if (!dest_insn) { 1579 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); 1580 1581 /* 1582 * retbleed_untrain_ret() jumps to 1583 * __x86_return_thunk(), but objtool can't find 1584 * the thunk's starting RET instruction, 1585 * because the RET is also in the middle of 1586 * another instruction. Objtool only knows 1587 * about the outer instruction. 1588 */ 1589 if (sym && sym->embedded_insn) { 1590 add_return_call(file, insn, false); 1591 continue; 1592 } 1593 1594 /* 1595 * GCOV/KCOV dead code can jump to the end of 1596 * the function/section. 1597 */ 1598 if (file->ignore_unreachables && func && 1599 dest_sec == insn->sec && 1600 dest_off == func->offset + func->len) 1601 continue; 1602 1603 ERROR_INSN(insn, "can't find jump dest instruction at %s", 1604 offstr(dest_sec, dest_off)); 1605 return -1; 1606 } 1607 1608 if (!dest_sym || is_sec_sym(dest_sym)) { 1609 dest_sym = dest_insn->sym; 1610 if (!dest_sym) 1611 goto set_jump_dest; 1612 } 1613 1614 if (dest_sym->retpoline_thunk && dest_insn->offset == dest_sym->offset) { 1615 if (add_retpoline_call(file, insn)) 1616 return -1; 1617 continue; 1618 } 1619 1620 if (dest_sym->return_thunk && dest_insn->offset == dest_sym->offset) { 1621 add_return_call(file, insn, true); 1622 continue; 1623 } 1624 1625 if (!insn->sym || insn->sym == dest_insn->sym) 1626 goto set_jump_dest; 1627 1628 /* 1629 * Internal cross-function jump. 1630 */ 1631 1632 /* 1633 * For GCC 8+, create parent/child links for any cold 1634 * subfunctions. This is _mostly_ redundant with a 1635 * similar initialization in read_symbols(). 1636 * 1637 * If a function has aliases, we want the *first* such 1638 * function in the symbol table to be the subfunction's 1639 * parent. In that case we overwrite the 1640 * initialization done in read_symbols(). 1641 * 1642 * However this code can't completely replace the 1643 * read_symbols() code because this doesn't detect the 1644 * case where the parent function's only reference to a 1645 * subfunction is through a jump table. 1646 */ 1647 if (func && dest_sym->cold) { 1648 func->cfunc = dest_sym; 1649 dest_sym->pfunc = func; 1650 goto set_jump_dest; 1651 } 1652 1653 if (is_first_func_insn(file, dest_insn)) { 1654 /* Internal sibling call */ 1655 if (add_call_dest(file, insn, dest_sym, true)) 1656 return -1; 1657 continue; 1658 } 1659 1660 set_jump_dest: 1661 insn->jump_dest = dest_insn; 1662 } 1663 1664 return 0; 1665 } 1666 1667 static struct symbol *find_call_destination(struct section *sec, unsigned long offset) 1668 { 1669 struct symbol *call_dest; 1670 1671 call_dest = find_func_by_offset(sec, offset); 1672 if (!call_dest) 1673 call_dest = find_symbol_by_offset(sec, offset); 1674 1675 return call_dest; 1676 } 1677 1678 /* 1679 * Find the destination instructions for all calls. 1680 */ 1681 static int add_call_destinations(struct objtool_file *file) 1682 { 1683 struct instruction *insn; 1684 unsigned long dest_off; 1685 struct symbol *dest; 1686 struct reloc *reloc; 1687 1688 for_each_insn(file, insn) { 1689 struct symbol *func = insn_func(insn); 1690 if (insn->type != INSN_CALL) 1691 continue; 1692 1693 reloc = insn_reloc(file, insn); 1694 if (!reloc) { 1695 dest_off = arch_jump_destination(insn); 1696 dest = find_call_destination(insn->sec, dest_off); 1697 1698 if (add_call_dest(file, insn, dest, false)) 1699 return -1; 1700 1701 if (func && func->ignore) 1702 continue; 1703 1704 if (!insn_call_dest(insn)) { 1705 ERROR_INSN(insn, "unannotated intra-function call"); 1706 return -1; 1707 } 1708 1709 if (func && !is_func_sym(insn_call_dest(insn))) { 1710 ERROR_INSN(insn, "unsupported call to non-function"); 1711 return -1; 1712 } 1713 1714 } else if (is_sec_sym(reloc->sym)) { 1715 dest_off = arch_insn_adjusted_addend(insn, reloc); 1716 dest = find_call_destination(reloc->sym->sec, dest_off); 1717 if (!dest) { 1718 ERROR_INSN(insn, "can't find call dest symbol at %s+0x%lx", 1719 reloc->sym->sec->name, dest_off); 1720 return -1; 1721 } 1722 1723 if (add_call_dest(file, insn, dest, false)) 1724 return -1; 1725 1726 } else if (reloc->sym->retpoline_thunk) { 1727 if (add_retpoline_call(file, insn)) 1728 return -1; 1729 1730 } else { 1731 if (add_call_dest(file, insn, reloc->sym, false)) 1732 return -1; 1733 } 1734 } 1735 1736 return 0; 1737 } 1738 1739 /* 1740 * The .alternatives section requires some extra special care over and above 1741 * other special sections because alternatives are patched in place. 1742 */ 1743 static int handle_group_alt(struct objtool_file *file, 1744 struct special_alt *special_alt, 1745 struct instruction *orig_insn, 1746 struct instruction **new_insn) 1747 { 1748 struct instruction *last_new_insn = NULL, *insn, *nop = NULL; 1749 struct alt_group *orig_alt_group, *new_alt_group; 1750 unsigned long dest_off; 1751 1752 orig_alt_group = orig_insn->alt_group; 1753 if (!orig_alt_group) { 1754 struct instruction *last_orig_insn = NULL; 1755 1756 orig_alt_group = calloc(1, sizeof(*orig_alt_group)); 1757 if (!orig_alt_group) { 1758 ERROR_GLIBC("calloc"); 1759 return -1; 1760 } 1761 orig_alt_group->cfi = calloc(special_alt->orig_len, 1762 sizeof(struct cfi_state *)); 1763 if (!orig_alt_group->cfi) { 1764 ERROR_GLIBC("calloc"); 1765 return -1; 1766 } 1767 1768 insn = orig_insn; 1769 sec_for_each_insn_from(file, insn) { 1770 if (insn->offset >= special_alt->orig_off + special_alt->orig_len) 1771 break; 1772 1773 insn->alt_group = orig_alt_group; 1774 last_orig_insn = insn; 1775 } 1776 orig_alt_group->orig_group = NULL; 1777 orig_alt_group->first_insn = orig_insn; 1778 orig_alt_group->last_insn = last_orig_insn; 1779 orig_alt_group->nop = NULL; 1780 orig_alt_group->ignore = orig_insn->ignore_alts; 1781 } else { 1782 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - 1783 orig_alt_group->first_insn->offset != special_alt->orig_len) { 1784 ERROR_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d", 1785 orig_alt_group->last_insn->offset + 1786 orig_alt_group->last_insn->len - 1787 orig_alt_group->first_insn->offset, 1788 special_alt->orig_len); 1789 return -1; 1790 } 1791 } 1792 1793 new_alt_group = calloc(1, sizeof(*new_alt_group)); 1794 if (!new_alt_group) { 1795 ERROR_GLIBC("calloc"); 1796 return -1; 1797 } 1798 1799 if (special_alt->new_len < special_alt->orig_len) { 1800 /* 1801 * Insert a fake nop at the end to make the replacement 1802 * alt_group the same size as the original. This is needed to 1803 * allow propagate_alt_cfi() to do its magic. When the last 1804 * instruction affects the stack, the instruction after it (the 1805 * nop) will propagate the new state to the shared CFI array. 1806 */ 1807 nop = calloc(1, sizeof(*nop)); 1808 if (!nop) { 1809 ERROR_GLIBC("calloc"); 1810 return -1; 1811 } 1812 memset(nop, 0, sizeof(*nop)); 1813 1814 nop->sec = special_alt->new_sec; 1815 nop->offset = special_alt->new_off + special_alt->new_len; 1816 nop->len = special_alt->orig_len - special_alt->new_len; 1817 nop->type = INSN_NOP; 1818 nop->sym = orig_insn->sym; 1819 nop->alt_group = new_alt_group; 1820 nop->fake = 1; 1821 } 1822 1823 if (!special_alt->new_len) { 1824 *new_insn = nop; 1825 goto end; 1826 } 1827 1828 insn = *new_insn; 1829 sec_for_each_insn_from(file, insn) { 1830 struct reloc *alt_reloc; 1831 1832 if (insn->offset >= special_alt->new_off + special_alt->new_len) 1833 break; 1834 1835 last_new_insn = insn; 1836 1837 insn->sym = orig_insn->sym; 1838 insn->alt_group = new_alt_group; 1839 1840 /* 1841 * Since alternative replacement code is copy/pasted by the 1842 * kernel after applying relocations, generally such code can't 1843 * have relative-address relocation references to outside the 1844 * .altinstr_replacement section, unless the arch's 1845 * alternatives code can adjust the relative offsets 1846 * accordingly. 1847 */ 1848 alt_reloc = insn_reloc(file, insn); 1849 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) && 1850 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { 1851 1852 ERROR_INSN(insn, "unsupported relocation in alternatives section"); 1853 return -1; 1854 } 1855 1856 if (!is_static_jump(insn)) 1857 continue; 1858 1859 if (!insn->immediate) 1860 continue; 1861 1862 dest_off = arch_jump_destination(insn); 1863 if (dest_off == special_alt->new_off + special_alt->new_len) { 1864 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn); 1865 if (!insn->jump_dest) { 1866 ERROR_INSN(insn, "can't find alternative jump destination"); 1867 return -1; 1868 } 1869 } 1870 } 1871 1872 if (!last_new_insn) { 1873 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1874 "can't find last new alternative instruction"); 1875 return -1; 1876 } 1877 1878 end: 1879 new_alt_group->orig_group = orig_alt_group; 1880 new_alt_group->first_insn = *new_insn; 1881 new_alt_group->last_insn = last_new_insn; 1882 new_alt_group->nop = nop; 1883 new_alt_group->ignore = (*new_insn)->ignore_alts; 1884 new_alt_group->cfi = orig_alt_group->cfi; 1885 return 0; 1886 } 1887 1888 /* 1889 * A jump table entry can either convert a nop to a jump or a jump to a nop. 1890 * If the original instruction is a jump, make the alt entry an effective nop 1891 * by just skipping the original instruction. 1892 */ 1893 static int handle_jump_alt(struct objtool_file *file, 1894 struct special_alt *special_alt, 1895 struct instruction *orig_insn, 1896 struct instruction **new_insn) 1897 { 1898 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && 1899 orig_insn->type != INSN_NOP) { 1900 1901 ERROR_INSN(orig_insn, "unsupported instruction at jump label"); 1902 return -1; 1903 } 1904 1905 if (opts.hack_jump_label && special_alt->key_addend & 2) { 1906 struct reloc *reloc = insn_reloc(file, orig_insn); 1907 1908 if (reloc) 1909 set_reloc_type(file->elf, reloc, R_NONE); 1910 1911 if (elf_write_insn(file->elf, orig_insn->sec, 1912 orig_insn->offset, orig_insn->len, 1913 arch_nop_insn(orig_insn->len))) { 1914 return -1; 1915 } 1916 1917 orig_insn->type = INSN_NOP; 1918 } 1919 1920 if (orig_insn->type == INSN_NOP) { 1921 if (orig_insn->len == 2) 1922 file->jl_nop_short++; 1923 else 1924 file->jl_nop_long++; 1925 1926 return 0; 1927 } 1928 1929 if (orig_insn->len == 2) 1930 file->jl_short++; 1931 else 1932 file->jl_long++; 1933 1934 *new_insn = next_insn_same_sec(file, orig_insn); 1935 return 0; 1936 } 1937 1938 /* 1939 * Read all the special sections which have alternate instructions which can be 1940 * patched in or redirected to at runtime. Each instruction having alternate 1941 * instruction(s) has them added to its insn->alts list, which will be 1942 * traversed in validate_branch(). 1943 */ 1944 static int add_special_section_alts(struct objtool_file *file) 1945 { 1946 struct list_head special_alts; 1947 struct instruction *orig_insn, *new_insn; 1948 struct special_alt *special_alt, *tmp; 1949 struct alternative *alt; 1950 1951 if (special_get_alts(file->elf, &special_alts)) 1952 return -1; 1953 1954 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) { 1955 1956 orig_insn = find_insn(file, special_alt->orig_sec, 1957 special_alt->orig_off); 1958 if (!orig_insn) { 1959 ERROR_FUNC(special_alt->orig_sec, special_alt->orig_off, 1960 "special: can't find orig instruction"); 1961 return -1; 1962 } 1963 1964 new_insn = NULL; 1965 if (!special_alt->group || special_alt->new_len) { 1966 new_insn = find_insn(file, special_alt->new_sec, 1967 special_alt->new_off); 1968 if (!new_insn) { 1969 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1970 "special: can't find new instruction"); 1971 return -1; 1972 } 1973 } 1974 1975 if (special_alt->group) { 1976 if (!special_alt->orig_len) { 1977 ERROR_INSN(orig_insn, "empty alternative entry"); 1978 continue; 1979 } 1980 1981 if (handle_group_alt(file, special_alt, orig_insn, &new_insn)) 1982 return -1; 1983 1984 } else if (special_alt->jump_or_nop) { 1985 if (handle_jump_alt(file, special_alt, orig_insn, &new_insn)) 1986 return -1; 1987 } 1988 1989 alt = calloc(1, sizeof(*alt)); 1990 if (!alt) { 1991 ERROR_GLIBC("calloc"); 1992 return -1; 1993 } 1994 1995 alt->insn = new_insn; 1996 alt->next = orig_insn->alts; 1997 orig_insn->alts = alt; 1998 1999 list_del(&special_alt->list); 2000 free(special_alt); 2001 } 2002 2003 if (opts.stats) { 2004 printf("jl\\\tNOP\tJMP\n"); 2005 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); 2006 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); 2007 } 2008 2009 return 0; 2010 } 2011 2012 __weak unsigned long arch_jump_table_sym_offset(struct reloc *reloc, struct reloc *table) 2013 { 2014 return reloc->sym->offset + reloc_addend(reloc); 2015 } 2016 2017 static int add_jump_table(struct objtool_file *file, struct instruction *insn) 2018 { 2019 unsigned long table_size = insn_jump_table_size(insn); 2020 struct symbol *pfunc = insn_func(insn)->pfunc; 2021 struct reloc *table = insn_jump_table(insn); 2022 struct instruction *dest_insn; 2023 unsigned int prev_offset = 0; 2024 struct reloc *reloc = table; 2025 struct alternative *alt; 2026 unsigned long sym_offset; 2027 2028 /* 2029 * Each @reloc is a switch table relocation which points to the target 2030 * instruction. 2031 */ 2032 for_each_reloc_from(table->sec, reloc) { 2033 2034 /* Check for the end of the table: */ 2035 if (table_size && reloc_offset(reloc) - reloc_offset(table) >= table_size) 2036 break; 2037 if (reloc != table && is_jump_table(reloc)) 2038 break; 2039 2040 /* Make sure the table entries are consecutive: */ 2041 if (prev_offset && reloc_offset(reloc) != prev_offset + arch_reloc_size(reloc)) 2042 break; 2043 2044 sym_offset = arch_jump_table_sym_offset(reloc, table); 2045 2046 /* Detect function pointers from contiguous objects: */ 2047 if (reloc->sym->sec == pfunc->sec && sym_offset == pfunc->offset) 2048 break; 2049 2050 /* 2051 * Clang sometimes leaves dangling unused jump table entries 2052 * which point to the end of the function. Ignore them. 2053 */ 2054 if (reloc->sym->sec == pfunc->sec && 2055 sym_offset == pfunc->offset + pfunc->len) 2056 goto next; 2057 2058 dest_insn = find_insn(file, reloc->sym->sec, sym_offset); 2059 if (!dest_insn) 2060 break; 2061 2062 /* Make sure the destination is in the same function: */ 2063 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc) 2064 break; 2065 2066 alt = calloc(1, sizeof(*alt)); 2067 if (!alt) { 2068 ERROR_GLIBC("calloc"); 2069 return -1; 2070 } 2071 2072 alt->insn = dest_insn; 2073 alt->next = insn->alts; 2074 insn->alts = alt; 2075 next: 2076 prev_offset = reloc_offset(reloc); 2077 } 2078 2079 if (!prev_offset) { 2080 ERROR_INSN(insn, "can't find switch jump table"); 2081 return -1; 2082 } 2083 2084 return 0; 2085 } 2086 2087 /* 2088 * find_jump_table() - Given a dynamic jump, find the switch jump table 2089 * associated with it. 2090 */ 2091 static void find_jump_table(struct objtool_file *file, struct symbol *func, 2092 struct instruction *insn) 2093 { 2094 struct reloc *table_reloc; 2095 struct instruction *dest_insn, *orig_insn = insn; 2096 unsigned long table_size; 2097 unsigned long sym_offset; 2098 2099 /* 2100 * Backward search using the @first_jump_src links, these help avoid 2101 * much of the 'in between' code. Which avoids us getting confused by 2102 * it. 2103 */ 2104 for (; 2105 insn && insn_func(insn) && insn_func(insn)->pfunc == func; 2106 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) { 2107 2108 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC) 2109 break; 2110 2111 /* allow small jumps within the range */ 2112 if (insn->type == INSN_JUMP_UNCONDITIONAL && 2113 insn->jump_dest && 2114 (insn->jump_dest->offset <= insn->offset || 2115 insn->jump_dest->offset > orig_insn->offset)) 2116 break; 2117 2118 table_reloc = arch_find_switch_table(file, insn, &table_size); 2119 if (!table_reloc) 2120 continue; 2121 2122 sym_offset = table_reloc->sym->offset + reloc_addend(table_reloc); 2123 2124 dest_insn = find_insn(file, table_reloc->sym->sec, sym_offset); 2125 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func) 2126 continue; 2127 2128 set_jump_table(table_reloc); 2129 orig_insn->_jump_table = table_reloc; 2130 orig_insn->_jump_table_size = table_size; 2131 2132 break; 2133 } 2134 } 2135 2136 /* 2137 * First pass: Mark the head of each jump table so that in the next pass, 2138 * we know when a given jump table ends and the next one starts. 2139 */ 2140 static void mark_func_jump_tables(struct objtool_file *file, 2141 struct symbol *func) 2142 { 2143 struct instruction *insn, *last = NULL; 2144 2145 func_for_each_insn(file, func, insn) { 2146 if (!last) 2147 last = insn; 2148 2149 /* 2150 * Store back-pointers for unconditional forward jumps such 2151 * that find_jump_table() can back-track using those and 2152 * avoid some potentially confusing code. 2153 */ 2154 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && 2155 insn->offset > last->offset && 2156 insn->jump_dest->offset > insn->offset && 2157 !insn->jump_dest->first_jump_src) { 2158 2159 insn->jump_dest->first_jump_src = insn; 2160 last = insn->jump_dest; 2161 } 2162 2163 if (insn->type != INSN_JUMP_DYNAMIC) 2164 continue; 2165 2166 find_jump_table(file, func, insn); 2167 } 2168 } 2169 2170 static int add_func_jump_tables(struct objtool_file *file, 2171 struct symbol *func) 2172 { 2173 struct instruction *insn; 2174 2175 func_for_each_insn(file, func, insn) { 2176 if (!insn_jump_table(insn)) 2177 continue; 2178 2179 if (add_jump_table(file, insn)) 2180 return -1; 2181 } 2182 2183 return 0; 2184 } 2185 2186 /* 2187 * For some switch statements, gcc generates a jump table in the .rodata 2188 * section which contains a list of addresses within the function to jump to. 2189 * This finds these jump tables and adds them to the insn->alts lists. 2190 */ 2191 static int add_jump_table_alts(struct objtool_file *file) 2192 { 2193 struct symbol *func; 2194 2195 if (!file->rodata) 2196 return 0; 2197 2198 for_each_sym(file->elf, func) { 2199 if (!is_func_sym(func)) 2200 continue; 2201 2202 mark_func_jump_tables(file, func); 2203 if (add_func_jump_tables(file, func)) 2204 return -1; 2205 } 2206 2207 return 0; 2208 } 2209 2210 static void set_func_state(struct cfi_state *state) 2211 { 2212 state->cfa = initial_func_cfi.cfa; 2213 memcpy(&state->regs, &initial_func_cfi.regs, 2214 CFI_NUM_REGS * sizeof(struct cfi_reg)); 2215 state->stack_size = initial_func_cfi.cfa.offset; 2216 state->type = UNWIND_HINT_TYPE_CALL; 2217 } 2218 2219 static int read_unwind_hints(struct objtool_file *file) 2220 { 2221 struct cfi_state cfi = init_cfi; 2222 struct section *sec; 2223 struct unwind_hint *hint; 2224 struct instruction *insn; 2225 struct reloc *reloc; 2226 unsigned long offset; 2227 int i; 2228 2229 sec = find_section_by_name(file->elf, ".discard.unwind_hints"); 2230 if (!sec) 2231 return 0; 2232 2233 if (!sec->rsec) { 2234 ERROR("missing .rela.discard.unwind_hints section"); 2235 return -1; 2236 } 2237 2238 if (sec_size(sec) % sizeof(struct unwind_hint)) { 2239 ERROR("struct unwind_hint size mismatch"); 2240 return -1; 2241 } 2242 2243 file->hints = true; 2244 2245 for (i = 0; i < sec_size(sec) / sizeof(struct unwind_hint); i++) { 2246 hint = (struct unwind_hint *)sec->data->d_buf + i; 2247 2248 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); 2249 if (!reloc) { 2250 ERROR("can't find reloc for unwind_hints[%d]", i); 2251 return -1; 2252 } 2253 2254 offset = reloc->sym->offset + reloc_addend(reloc); 2255 2256 insn = find_insn(file, reloc->sym->sec, offset); 2257 if (!insn) { 2258 ERROR("can't find insn for unwind_hints[%d]", i); 2259 return -1; 2260 } 2261 2262 insn->hint = true; 2263 2264 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) { 2265 insn->cfi = &force_undefined_cfi; 2266 continue; 2267 } 2268 2269 if (hint->type == UNWIND_HINT_TYPE_SAVE) { 2270 insn->hint = false; 2271 insn->save = true; 2272 continue; 2273 } 2274 2275 if (hint->type == UNWIND_HINT_TYPE_RESTORE) { 2276 insn->restore = true; 2277 continue; 2278 } 2279 2280 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 2281 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 2282 2283 if (sym && is_global_sym(sym)) { 2284 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { 2285 ERROR_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR"); 2286 return -1; 2287 } 2288 } 2289 } 2290 2291 if (hint->type == UNWIND_HINT_TYPE_FUNC) { 2292 insn->cfi = &func_cfi; 2293 continue; 2294 } 2295 2296 if (insn->cfi) 2297 cfi = *(insn->cfi); 2298 2299 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { 2300 ERROR_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg); 2301 return -1; 2302 } 2303 2304 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); 2305 cfi.type = hint->type; 2306 cfi.signal = hint->signal; 2307 2308 insn->cfi = cfi_hash_find_or_add(&cfi); 2309 } 2310 2311 return 0; 2312 } 2313 2314 static int read_annotate(struct objtool_file *file, 2315 int (*func)(struct objtool_file *file, int type, struct instruction *insn)) 2316 { 2317 struct section *sec; 2318 struct instruction *insn; 2319 struct reloc *reloc; 2320 uint64_t offset; 2321 int type; 2322 2323 sec = find_section_by_name(file->elf, ".discard.annotate_insn"); 2324 if (!sec) 2325 return 0; 2326 2327 if (!sec->rsec) 2328 return 0; 2329 2330 if (sec->sh.sh_entsize != 8) { 2331 static bool warned = false; 2332 if (!warned && opts.verbose) { 2333 WARN("%s: dodgy linker, sh_entsize != 8", sec->name); 2334 warned = true; 2335 } 2336 sec->sh.sh_entsize = 8; 2337 } 2338 2339 if (sec_num_entries(sec) != sec_num_entries(sec->rsec)) { 2340 ERROR("bad .discard.annotate_insn section: missing relocs"); 2341 return -1; 2342 } 2343 2344 for_each_reloc(sec->rsec, reloc) { 2345 type = annotype(file->elf, sec, reloc); 2346 offset = reloc->sym->offset + reloc_addend(reloc); 2347 insn = find_insn(file, reloc->sym->sec, offset); 2348 2349 if (!insn) { 2350 ERROR("bad .discard.annotate_insn entry: %d of type %d", reloc_idx(reloc), type); 2351 return -1; 2352 } 2353 2354 if (func(file, type, insn)) 2355 return -1; 2356 } 2357 2358 return 0; 2359 } 2360 2361 static int __annotate_early(struct objtool_file *file, int type, struct instruction *insn) 2362 { 2363 switch (type) { 2364 2365 /* Must be before add_special_section_alts() */ 2366 case ANNOTYPE_IGNORE_ALTS: 2367 insn->ignore_alts = true; 2368 break; 2369 2370 /* 2371 * Must be before read_unwind_hints() since that needs insn->noendbr. 2372 */ 2373 case ANNOTYPE_NOENDBR: 2374 insn->noendbr = 1; 2375 break; 2376 2377 default: 2378 break; 2379 } 2380 2381 return 0; 2382 } 2383 2384 static int __annotate_ifc(struct objtool_file *file, int type, struct instruction *insn) 2385 { 2386 unsigned long dest_off; 2387 2388 if (type != ANNOTYPE_INTRA_FUNCTION_CALL) 2389 return 0; 2390 2391 if (insn->type != INSN_CALL) { 2392 ERROR_INSN(insn, "intra_function_call not a direct call"); 2393 return -1; 2394 } 2395 2396 /* 2397 * Treat intra-function CALLs as JMPs, but with a stack_op. 2398 * See add_call_destinations(), which strips stack_ops from 2399 * normal CALLs. 2400 */ 2401 insn->type = INSN_JUMP_UNCONDITIONAL; 2402 2403 dest_off = arch_jump_destination(insn); 2404 insn->jump_dest = find_insn(file, insn->sec, dest_off); 2405 if (!insn->jump_dest) { 2406 ERROR_INSN(insn, "can't find call dest at %s+0x%lx", 2407 insn->sec->name, dest_off); 2408 return -1; 2409 } 2410 2411 return 0; 2412 } 2413 2414 static int __annotate_late(struct objtool_file *file, int type, struct instruction *insn) 2415 { 2416 struct symbol *sym; 2417 2418 switch (type) { 2419 case ANNOTYPE_NOENDBR: 2420 /* early */ 2421 break; 2422 2423 case ANNOTYPE_RETPOLINE_SAFE: 2424 if (insn->type != INSN_JUMP_DYNAMIC && 2425 insn->type != INSN_CALL_DYNAMIC && 2426 insn->type != INSN_RETURN && 2427 insn->type != INSN_NOP) { 2428 ERROR_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); 2429 return -1; 2430 } 2431 2432 insn->retpoline_safe = true; 2433 break; 2434 2435 case ANNOTYPE_INSTR_BEGIN: 2436 insn->instr++; 2437 break; 2438 2439 case ANNOTYPE_INSTR_END: 2440 insn->instr--; 2441 break; 2442 2443 case ANNOTYPE_UNRET_BEGIN: 2444 insn->unret = 1; 2445 break; 2446 2447 case ANNOTYPE_IGNORE_ALTS: 2448 /* early */ 2449 break; 2450 2451 case ANNOTYPE_INTRA_FUNCTION_CALL: 2452 /* ifc */ 2453 break; 2454 2455 case ANNOTYPE_REACHABLE: 2456 insn->dead_end = false; 2457 break; 2458 2459 case ANNOTYPE_NOCFI: 2460 sym = insn->sym; 2461 if (!sym) { 2462 ERROR_INSN(insn, "dodgy NOCFI annotation"); 2463 return -1; 2464 } 2465 insn->sym->nocfi = 1; 2466 break; 2467 2468 default: 2469 ERROR_INSN(insn, "Unknown annotation type: %d", type); 2470 return -1; 2471 } 2472 2473 return 0; 2474 } 2475 2476 /* 2477 * Return true if name matches an instrumentation function, where calls to that 2478 * function from noinstr code can safely be removed, but compilers won't do so. 2479 */ 2480 static bool is_profiling_func(const char *name) 2481 { 2482 /* 2483 * Many compilers cannot disable KCOV with a function attribute. 2484 */ 2485 if (!strncmp(name, "__sanitizer_cov_", 16)) 2486 return true; 2487 2488 return false; 2489 } 2490 2491 static int classify_symbols(struct objtool_file *file) 2492 { 2493 struct symbol *func; 2494 2495 for_each_sym(file->elf, func) { 2496 if (is_notype_sym(func) && strstarts(func->name, ".L")) 2497 func->local_label = true; 2498 2499 if (!is_global_sym(func)) 2500 continue; 2501 2502 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, 2503 strlen(STATIC_CALL_TRAMP_PREFIX_STR))) 2504 func->static_call_tramp = true; 2505 2506 if (arch_is_retpoline(func)) 2507 func->retpoline_thunk = true; 2508 2509 if (arch_is_rethunk(func)) 2510 func->return_thunk = true; 2511 2512 if (arch_is_embedded_insn(func)) 2513 func->embedded_insn = true; 2514 2515 if (arch_ftrace_match(func->name)) 2516 func->fentry = true; 2517 2518 if (is_profiling_func(func->name)) 2519 func->profiling_func = true; 2520 } 2521 2522 return 0; 2523 } 2524 2525 static void mark_rodata(struct objtool_file *file) 2526 { 2527 struct section *sec; 2528 bool found = false; 2529 2530 /* 2531 * Search for the following rodata sections, each of which can 2532 * potentially contain jump tables: 2533 * 2534 * - .rodata: can contain GCC switch tables 2535 * - .rodata.<func>: same, if -fdata-sections is being used 2536 * - .data.rel.ro.c_jump_table: contains C annotated jump tables 2537 * 2538 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2539 */ 2540 for_each_sec(file->elf, sec) { 2541 if ((!strncmp(sec->name, ".rodata", 7) && 2542 !strstr(sec->name, ".str1.")) || 2543 !strncmp(sec->name, ".data.rel.ro", 12)) { 2544 sec->rodata = true; 2545 found = true; 2546 } 2547 } 2548 2549 file->rodata = found; 2550 } 2551 2552 static void mark_holes(struct objtool_file *file) 2553 { 2554 struct instruction *insn; 2555 bool in_hole = false; 2556 2557 if (!opts.link) 2558 return; 2559 2560 /* 2561 * Whole archive runs might encounter dead code from weak symbols. 2562 * This is where the linker will have dropped the weak symbol in 2563 * favour of a regular symbol, but leaves the code in place. 2564 */ 2565 for_each_insn(file, insn) { 2566 if (insn->sym || !find_symbol_hole_containing(insn->sec, insn->offset)) { 2567 in_hole = false; 2568 continue; 2569 } 2570 2571 /* Skip function padding and pfx code */ 2572 if (!in_hole && insn->type == INSN_NOP) 2573 continue; 2574 2575 in_hole = true; 2576 insn->hole = 1; 2577 2578 /* 2579 * If this hole jumps to a .cold function, mark it ignore. 2580 */ 2581 if (insn->jump_dest) { 2582 struct symbol *dest_func = insn_func(insn->jump_dest); 2583 2584 if (dest_func && dest_func->cold) 2585 dest_func->ignore = true; 2586 } 2587 } 2588 } 2589 2590 static bool validate_branch_enabled(void) 2591 { 2592 return opts.stackval || 2593 opts.orc || 2594 opts.uaccess || 2595 opts.checksum; 2596 } 2597 2598 static int decode_sections(struct objtool_file *file) 2599 { 2600 file->klp = is_livepatch_module(file); 2601 2602 mark_rodata(file); 2603 2604 if (init_pv_ops(file)) 2605 return -1; 2606 2607 /* 2608 * Must be before add_{jump_call}_destination. 2609 */ 2610 if (classify_symbols(file)) 2611 return -1; 2612 2613 if (decode_instructions(file)) 2614 return -1; 2615 2616 if (add_ignores(file)) 2617 return -1; 2618 2619 add_uaccess_safe(file); 2620 2621 if (read_annotate(file, __annotate_early)) 2622 return -1; 2623 2624 /* 2625 * Must be before add_jump_destinations(), which depends on 'func' 2626 * being set for alternatives, to enable proper sibling call detection. 2627 */ 2628 if (validate_branch_enabled() || opts.noinstr || opts.hack_jump_label) { 2629 if (add_special_section_alts(file)) 2630 return -1; 2631 } 2632 2633 if (add_jump_destinations(file)) 2634 return -1; 2635 2636 /* 2637 * Must be before add_call_destination(); it changes INSN_CALL to 2638 * INSN_JUMP. 2639 */ 2640 if (read_annotate(file, __annotate_ifc)) 2641 return -1; 2642 2643 if (add_call_destinations(file)) 2644 return -1; 2645 2646 if (add_jump_table_alts(file)) 2647 return -1; 2648 2649 if (read_unwind_hints(file)) 2650 return -1; 2651 2652 /* Must be after add_jump_destinations() */ 2653 mark_holes(file); 2654 2655 /* 2656 * Must be after add_call_destinations() such that it can override 2657 * dead_end_function() marks. 2658 */ 2659 if (read_annotate(file, __annotate_late)) 2660 return -1; 2661 2662 return 0; 2663 } 2664 2665 static bool is_special_call(struct instruction *insn) 2666 { 2667 if (insn->type == INSN_CALL) { 2668 struct symbol *dest = insn_call_dest(insn); 2669 2670 if (!dest) 2671 return false; 2672 2673 if (dest->fentry || dest->embedded_insn) 2674 return true; 2675 } 2676 2677 return false; 2678 } 2679 2680 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state) 2681 { 2682 struct cfi_state *cfi = &state->cfi; 2683 int i; 2684 2685 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap) 2686 return true; 2687 2688 if (cfi->cfa.offset != initial_func_cfi.cfa.offset) 2689 return true; 2690 2691 if (cfi->stack_size != initial_func_cfi.cfa.offset) 2692 return true; 2693 2694 for (i = 0; i < CFI_NUM_REGS; i++) { 2695 if (cfi->regs[i].base != initial_func_cfi.regs[i].base || 2696 cfi->regs[i].offset != initial_func_cfi.regs[i].offset) 2697 return true; 2698 } 2699 2700 return false; 2701 } 2702 2703 static bool check_reg_frame_pos(const struct cfi_reg *reg, 2704 int expected_offset) 2705 { 2706 return reg->base == CFI_CFA && 2707 reg->offset == expected_offset; 2708 } 2709 2710 static bool has_valid_stack_frame(struct insn_state *state) 2711 { 2712 struct cfi_state *cfi = &state->cfi; 2713 2714 if (cfi->cfa.base == CFI_BP && 2715 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) && 2716 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8)) 2717 return true; 2718 2719 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP) 2720 return true; 2721 2722 return false; 2723 } 2724 2725 static int update_cfi_state_regs(struct instruction *insn, 2726 struct cfi_state *cfi, 2727 struct stack_op *op) 2728 { 2729 struct cfi_reg *cfa = &cfi->cfa; 2730 2731 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT) 2732 return 0; 2733 2734 /* push */ 2735 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF) 2736 cfa->offset += 8; 2737 2738 /* pop */ 2739 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF) 2740 cfa->offset -= 8; 2741 2742 /* add immediate to sp */ 2743 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD && 2744 op->dest.reg == CFI_SP && op->src.reg == CFI_SP) 2745 cfa->offset -= op->src.offset; 2746 2747 return 0; 2748 } 2749 2750 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset) 2751 { 2752 if (arch_callee_saved_reg(reg) && 2753 cfi->regs[reg].base == CFI_UNDEFINED) { 2754 cfi->regs[reg].base = base; 2755 cfi->regs[reg].offset = offset; 2756 } 2757 } 2758 2759 static void restore_reg(struct cfi_state *cfi, unsigned char reg) 2760 { 2761 cfi->regs[reg].base = initial_func_cfi.regs[reg].base; 2762 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset; 2763 } 2764 2765 /* 2766 * A note about DRAP stack alignment: 2767 * 2768 * GCC has the concept of a DRAP register, which is used to help keep track of 2769 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP 2770 * register. The typical DRAP pattern is: 2771 * 2772 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10 2773 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp 2774 * 41 ff 72 f8 pushq -0x8(%r10) 2775 * 55 push %rbp 2776 * 48 89 e5 mov %rsp,%rbp 2777 * (more pushes) 2778 * 41 52 push %r10 2779 * ... 2780 * 41 5a pop %r10 2781 * (more pops) 2782 * 5d pop %rbp 2783 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2784 * c3 retq 2785 * 2786 * There are some variations in the epilogues, like: 2787 * 2788 * 5b pop %rbx 2789 * 41 5a pop %r10 2790 * 41 5c pop %r12 2791 * 41 5d pop %r13 2792 * 41 5e pop %r14 2793 * c9 leaveq 2794 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2795 * c3 retq 2796 * 2797 * and: 2798 * 2799 * 4c 8b 55 e8 mov -0x18(%rbp),%r10 2800 * 48 8b 5d e0 mov -0x20(%rbp),%rbx 2801 * 4c 8b 65 f0 mov -0x10(%rbp),%r12 2802 * 4c 8b 6d f8 mov -0x8(%rbp),%r13 2803 * c9 leaveq 2804 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2805 * c3 retq 2806 * 2807 * Sometimes r13 is used as the DRAP register, in which case it's saved and 2808 * restored beforehand: 2809 * 2810 * 41 55 push %r13 2811 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13 2812 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp 2813 * ... 2814 * 49 8d 65 f0 lea -0x10(%r13),%rsp 2815 * 41 5d pop %r13 2816 * c3 retq 2817 */ 2818 static int update_cfi_state(struct instruction *insn, 2819 struct instruction *next_insn, 2820 struct cfi_state *cfi, struct stack_op *op) 2821 { 2822 struct cfi_reg *cfa = &cfi->cfa; 2823 struct cfi_reg *regs = cfi->regs; 2824 2825 /* ignore UNWIND_HINT_UNDEFINED regions */ 2826 if (cfi->force_undefined) 2827 return 0; 2828 2829 /* stack operations don't make sense with an undefined CFA */ 2830 if (cfa->base == CFI_UNDEFINED) { 2831 if (insn_func(insn)) { 2832 WARN_INSN(insn, "undefined stack state"); 2833 return 1; 2834 } 2835 return 0; 2836 } 2837 2838 if (cfi->type == UNWIND_HINT_TYPE_REGS || 2839 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) 2840 return update_cfi_state_regs(insn, cfi, op); 2841 2842 switch (op->dest.type) { 2843 2844 case OP_DEST_REG: 2845 switch (op->src.type) { 2846 2847 case OP_SRC_REG: 2848 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP && 2849 cfa->base == CFI_SP && 2850 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) { 2851 2852 /* mov %rsp, %rbp */ 2853 cfa->base = op->dest.reg; 2854 cfi->bp_scratch = false; 2855 } 2856 2857 else if (op->src.reg == CFI_SP && 2858 op->dest.reg == CFI_BP && cfi->drap) { 2859 2860 /* drap: mov %rsp, %rbp */ 2861 regs[CFI_BP].base = CFI_BP; 2862 regs[CFI_BP].offset = -cfi->stack_size; 2863 cfi->bp_scratch = false; 2864 } 2865 2866 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2867 2868 /* 2869 * mov %rsp, %reg 2870 * 2871 * This is needed for the rare case where GCC 2872 * does: 2873 * 2874 * mov %rsp, %rax 2875 * ... 2876 * mov %rax, %rsp 2877 */ 2878 cfi->vals[op->dest.reg].base = CFI_CFA; 2879 cfi->vals[op->dest.reg].offset = -cfi->stack_size; 2880 } 2881 2882 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP && 2883 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) { 2884 2885 /* 2886 * mov %rbp, %rsp 2887 * 2888 * Restore the original stack pointer (Clang). 2889 */ 2890 cfi->stack_size = -cfi->regs[CFI_BP].offset; 2891 } 2892 2893 else if (op->dest.reg == cfa->base) { 2894 2895 /* mov %reg, %rsp */ 2896 if (cfa->base == CFI_SP && 2897 cfi->vals[op->src.reg].base == CFI_CFA) { 2898 2899 /* 2900 * This is needed for the rare case 2901 * where GCC does something dumb like: 2902 * 2903 * lea 0x8(%rsp), %rcx 2904 * ... 2905 * mov %rcx, %rsp 2906 */ 2907 cfa->offset = -cfi->vals[op->src.reg].offset; 2908 cfi->stack_size = cfa->offset; 2909 2910 } else if (cfa->base == CFI_SP && 2911 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2912 cfi->vals[op->src.reg].offset == cfa->offset) { 2913 2914 /* 2915 * Stack swizzle: 2916 * 2917 * 1: mov %rsp, (%[tos]) 2918 * 2: mov %[tos], %rsp 2919 * ... 2920 * 3: pop %rsp 2921 * 2922 * Where: 2923 * 2924 * 1 - places a pointer to the previous 2925 * stack at the Top-of-Stack of the 2926 * new stack. 2927 * 2928 * 2 - switches to the new stack. 2929 * 2930 * 3 - pops the Top-of-Stack to restore 2931 * the original stack. 2932 * 2933 * Note: we set base to SP_INDIRECT 2934 * here and preserve offset. Therefore 2935 * when the unwinder reaches ToS it 2936 * will dereference SP and then add the 2937 * offset to find the next frame, IOW: 2938 * (%rsp) + offset. 2939 */ 2940 cfa->base = CFI_SP_INDIRECT; 2941 2942 } else { 2943 cfa->base = CFI_UNDEFINED; 2944 cfa->offset = 0; 2945 } 2946 } 2947 2948 else if (op->dest.reg == CFI_SP && 2949 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2950 cfi->vals[op->src.reg].offset == cfa->offset) { 2951 2952 /* 2953 * The same stack swizzle case 2) as above. But 2954 * because we can't change cfa->base, case 3) 2955 * will become a regular POP. Pretend we're a 2956 * PUSH so things don't go unbalanced. 2957 */ 2958 cfi->stack_size += 8; 2959 } 2960 2961 2962 break; 2963 2964 case OP_SRC_ADD: 2965 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) { 2966 2967 /* add imm, %rsp */ 2968 cfi->stack_size -= op->src.offset; 2969 if (cfa->base == CFI_SP) 2970 cfa->offset -= op->src.offset; 2971 break; 2972 } 2973 2974 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP && 2975 insn->sym->frame_pointer) { 2976 /* addi.d fp,sp,imm on LoongArch */ 2977 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) { 2978 cfa->base = CFI_BP; 2979 cfa->offset = 0; 2980 } 2981 break; 2982 } 2983 2984 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { 2985 /* addi.d sp,fp,imm on LoongArch */ 2986 if (cfa->base == CFI_BP && cfa->offset == 0) { 2987 if (insn->sym->frame_pointer) { 2988 cfa->base = CFI_SP; 2989 cfa->offset = -op->src.offset; 2990 } 2991 } else { 2992 /* lea disp(%rbp), %rsp */ 2993 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); 2994 } 2995 break; 2996 } 2997 2998 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2999 3000 /* drap: lea disp(%rsp), %drap */ 3001 cfi->drap_reg = op->dest.reg; 3002 3003 /* 3004 * lea disp(%rsp), %reg 3005 * 3006 * This is needed for the rare case where GCC 3007 * does something dumb like: 3008 * 3009 * lea 0x8(%rsp), %rcx 3010 * ... 3011 * mov %rcx, %rsp 3012 */ 3013 cfi->vals[op->dest.reg].base = CFI_CFA; 3014 cfi->vals[op->dest.reg].offset = \ 3015 -cfi->stack_size + op->src.offset; 3016 3017 break; 3018 } 3019 3020 if (cfi->drap && op->dest.reg == CFI_SP && 3021 op->src.reg == cfi->drap_reg) { 3022 3023 /* drap: lea disp(%drap), %rsp */ 3024 cfa->base = CFI_SP; 3025 cfa->offset = cfi->stack_size = -op->src.offset; 3026 cfi->drap_reg = CFI_UNDEFINED; 3027 cfi->drap = false; 3028 break; 3029 } 3030 3031 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { 3032 WARN_INSN(insn, "unsupported stack register modification"); 3033 return -1; 3034 } 3035 3036 break; 3037 3038 case OP_SRC_AND: 3039 if (op->dest.reg != CFI_SP || 3040 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || 3041 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { 3042 WARN_INSN(insn, "unsupported stack pointer realignment"); 3043 return -1; 3044 } 3045 3046 if (cfi->drap_reg != CFI_UNDEFINED) { 3047 /* drap: and imm, %rsp */ 3048 cfa->base = cfi->drap_reg; 3049 cfa->offset = cfi->stack_size = 0; 3050 cfi->drap = true; 3051 } 3052 3053 /* 3054 * Older versions of GCC (4.8ish) realign the stack 3055 * without DRAP, with a frame pointer. 3056 */ 3057 3058 break; 3059 3060 case OP_SRC_POP: 3061 case OP_SRC_POPF: 3062 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) { 3063 3064 /* pop %rsp; # restore from a stack swizzle */ 3065 cfa->base = CFI_SP; 3066 break; 3067 } 3068 3069 if (!cfi->drap && op->dest.reg == cfa->base) { 3070 3071 /* pop %rbp */ 3072 cfa->base = CFI_SP; 3073 } 3074 3075 if (cfi->drap && cfa->base == CFI_BP_INDIRECT && 3076 op->dest.reg == cfi->drap_reg && 3077 cfi->drap_offset == -cfi->stack_size) { 3078 3079 /* drap: pop %drap */ 3080 cfa->base = cfi->drap_reg; 3081 cfa->offset = 0; 3082 cfi->drap_offset = -1; 3083 3084 } else if (cfi->stack_size == -regs[op->dest.reg].offset) { 3085 3086 /* pop %reg */ 3087 restore_reg(cfi, op->dest.reg); 3088 } 3089 3090 cfi->stack_size -= 8; 3091 if (cfa->base == CFI_SP) 3092 cfa->offset -= 8; 3093 3094 break; 3095 3096 case OP_SRC_REG_INDIRECT: 3097 if (!cfi->drap && op->dest.reg == cfa->base && 3098 op->dest.reg == CFI_BP) { 3099 3100 /* mov disp(%rsp), %rbp */ 3101 cfa->base = CFI_SP; 3102 cfa->offset = cfi->stack_size; 3103 } 3104 3105 if (cfi->drap && op->src.reg == CFI_BP && 3106 op->src.offset == cfi->drap_offset) { 3107 3108 /* drap: mov disp(%rbp), %drap */ 3109 cfa->base = cfi->drap_reg; 3110 cfa->offset = 0; 3111 cfi->drap_offset = -1; 3112 } 3113 3114 if (cfi->drap && op->src.reg == CFI_BP && 3115 op->src.offset == regs[op->dest.reg].offset) { 3116 3117 /* drap: mov disp(%rbp), %reg */ 3118 restore_reg(cfi, op->dest.reg); 3119 3120 } else if (op->src.reg == cfa->base && 3121 op->src.offset == regs[op->dest.reg].offset + cfa->offset) { 3122 3123 /* mov disp(%rbp), %reg */ 3124 /* mov disp(%rsp), %reg */ 3125 restore_reg(cfi, op->dest.reg); 3126 3127 } else if (op->src.reg == CFI_SP && 3128 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) { 3129 3130 /* mov disp(%rsp), %reg */ 3131 restore_reg(cfi, op->dest.reg); 3132 } 3133 3134 break; 3135 3136 default: 3137 WARN_INSN(insn, "unknown stack-related instruction"); 3138 return -1; 3139 } 3140 3141 break; 3142 3143 case OP_DEST_PUSH: 3144 case OP_DEST_PUSHF: 3145 cfi->stack_size += 8; 3146 if (cfa->base == CFI_SP) 3147 cfa->offset += 8; 3148 3149 if (op->src.type != OP_SRC_REG) 3150 break; 3151 3152 if (cfi->drap) { 3153 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3154 3155 /* drap: push %drap */ 3156 cfa->base = CFI_BP_INDIRECT; 3157 cfa->offset = -cfi->stack_size; 3158 3159 /* save drap so we know when to restore it */ 3160 cfi->drap_offset = -cfi->stack_size; 3161 3162 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) { 3163 3164 /* drap: push %rbp */ 3165 cfi->stack_size = 0; 3166 3167 } else { 3168 3169 /* drap: push %reg */ 3170 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); 3171 } 3172 3173 } else { 3174 3175 /* push %reg */ 3176 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size); 3177 } 3178 3179 /* detect when asm code uses rbp as a scratch register */ 3180 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP && 3181 cfa->base != CFI_BP) 3182 cfi->bp_scratch = true; 3183 break; 3184 3185 case OP_DEST_REG_INDIRECT: 3186 3187 if (cfi->drap) { 3188 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3189 3190 /* drap: mov %drap, disp(%rbp) */ 3191 cfa->base = CFI_BP_INDIRECT; 3192 cfa->offset = op->dest.offset; 3193 3194 /* save drap offset so we know when to restore it */ 3195 cfi->drap_offset = op->dest.offset; 3196 } else { 3197 3198 /* drap: mov reg, disp(%rbp) */ 3199 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); 3200 } 3201 3202 } else if (op->dest.reg == cfa->base) { 3203 3204 /* mov reg, disp(%rbp) */ 3205 /* mov reg, disp(%rsp) */ 3206 save_reg(cfi, op->src.reg, CFI_CFA, 3207 op->dest.offset - cfi->cfa.offset); 3208 3209 } else if (op->dest.reg == CFI_SP) { 3210 3211 /* mov reg, disp(%rsp) */ 3212 save_reg(cfi, op->src.reg, CFI_CFA, 3213 op->dest.offset - cfi->stack_size); 3214 3215 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) { 3216 3217 /* mov %rsp, (%reg); # setup a stack swizzle. */ 3218 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT; 3219 cfi->vals[op->dest.reg].offset = cfa->offset; 3220 } 3221 3222 break; 3223 3224 case OP_DEST_MEM: 3225 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { 3226 WARN_INSN(insn, "unknown stack-related memory operation"); 3227 return -1; 3228 } 3229 3230 /* pop mem */ 3231 cfi->stack_size -= 8; 3232 if (cfa->base == CFI_SP) 3233 cfa->offset -= 8; 3234 3235 break; 3236 3237 default: 3238 WARN_INSN(insn, "unknown stack-related instruction"); 3239 return -1; 3240 } 3241 3242 return 0; 3243 } 3244 3245 /* 3246 * The stack layouts of alternatives instructions can sometimes diverge when 3247 * they have stack modifications. That's fine as long as the potential stack 3248 * layouts don't conflict at any given potential instruction boundary. 3249 * 3250 * Flatten the CFIs of the different alternative code streams (both original 3251 * and replacement) into a single shared CFI array which can be used to detect 3252 * conflicts and nicely feed a linear array of ORC entries to the unwinder. 3253 */ 3254 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn) 3255 { 3256 struct cfi_state **alt_cfi; 3257 int group_off; 3258 3259 if (!insn->alt_group) 3260 return 0; 3261 3262 if (!insn->cfi) { 3263 WARN("CFI missing"); 3264 return -1; 3265 } 3266 3267 alt_cfi = insn->alt_group->cfi; 3268 group_off = insn->offset - insn->alt_group->first_insn->offset; 3269 3270 if (!alt_cfi[group_off]) { 3271 alt_cfi[group_off] = insn->cfi; 3272 } else { 3273 if (cficmp(alt_cfi[group_off], insn->cfi)) { 3274 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group; 3275 struct instruction *orig = orig_group->first_insn; 3276 WARN_INSN(orig, "stack layout conflict in alternatives: %s", 3277 offstr(insn->sec, insn->offset)); 3278 return -1; 3279 } 3280 } 3281 3282 return 0; 3283 } 3284 3285 static int handle_insn_ops(struct instruction *insn, 3286 struct instruction *next_insn, 3287 struct insn_state *state) 3288 { 3289 struct stack_op *op; 3290 int ret; 3291 3292 for (op = insn->stack_ops; op; op = op->next) { 3293 3294 ret = update_cfi_state(insn, next_insn, &state->cfi, op); 3295 if (ret) 3296 return ret; 3297 3298 if (!opts.uaccess || !insn->alt_group) 3299 continue; 3300 3301 if (op->dest.type == OP_DEST_PUSHF) { 3302 if (!state->uaccess_stack) { 3303 state->uaccess_stack = 1; 3304 } else if (state->uaccess_stack >> 31) { 3305 WARN_INSN(insn, "PUSHF stack exhausted"); 3306 return 1; 3307 } 3308 state->uaccess_stack <<= 1; 3309 state->uaccess_stack |= state->uaccess; 3310 } 3311 3312 if (op->src.type == OP_SRC_POPF) { 3313 if (state->uaccess_stack) { 3314 state->uaccess = state->uaccess_stack & 1; 3315 state->uaccess_stack >>= 1; 3316 if (state->uaccess_stack == 1) 3317 state->uaccess_stack = 0; 3318 } 3319 } 3320 } 3321 3322 return 0; 3323 } 3324 3325 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) 3326 { 3327 struct cfi_state *cfi1 = insn->cfi; 3328 int i; 3329 3330 if (!cfi1) { 3331 WARN("CFI missing"); 3332 return false; 3333 } 3334 3335 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { 3336 3337 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d", 3338 cfi1->cfa.base, cfi1->cfa.offset, 3339 cfi2->cfa.base, cfi2->cfa.offset); 3340 return false; 3341 3342 } 3343 3344 if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) { 3345 for (i = 0; i < CFI_NUM_REGS; i++) { 3346 3347 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], sizeof(struct cfi_reg))) 3348 continue; 3349 3350 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", 3351 i, cfi1->regs[i].base, cfi1->regs[i].offset, 3352 i, cfi2->regs[i].base, cfi2->regs[i].offset); 3353 } 3354 return false; 3355 } 3356 3357 if (cfi1->type != cfi2->type) { 3358 3359 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d", 3360 cfi1->type, cfi2->type); 3361 return false; 3362 } 3363 3364 if (cfi1->drap != cfi2->drap || 3365 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || 3366 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { 3367 3368 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", 3369 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, 3370 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); 3371 return false; 3372 } 3373 3374 return true; 3375 } 3376 3377 static inline bool func_uaccess_safe(struct symbol *func) 3378 { 3379 if (func) 3380 return func->uaccess_safe; 3381 3382 return false; 3383 } 3384 3385 static inline const char *call_dest_name(struct instruction *insn) 3386 { 3387 static char pvname[19]; 3388 struct reloc *reloc; 3389 int idx; 3390 3391 if (insn_call_dest(insn)) 3392 return insn_call_dest(insn)->name; 3393 3394 reloc = insn_reloc(NULL, insn); 3395 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) { 3396 idx = (reloc_addend(reloc) / sizeof(void *)); 3397 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx); 3398 return pvname; 3399 } 3400 3401 return "{dynamic}"; 3402 } 3403 3404 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn) 3405 { 3406 struct symbol *target; 3407 struct reloc *reloc; 3408 int idx; 3409 3410 reloc = insn_reloc(file, insn); 3411 if (!reloc || strcmp(reloc->sym->name, "pv_ops")) 3412 return false; 3413 3414 idx = arch_insn_adjusted_addend(insn, reloc) / sizeof(void *); 3415 3416 if (file->pv_ops[idx].clean) 3417 return true; 3418 3419 file->pv_ops[idx].clean = true; 3420 3421 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) { 3422 if (!target->sec->noinstr) { 3423 WARN("pv_ops[%d]: %s", idx, target->name); 3424 file->pv_ops[idx].clean = false; 3425 } 3426 } 3427 3428 return file->pv_ops[idx].clean; 3429 } 3430 3431 static inline bool noinstr_call_dest(struct objtool_file *file, 3432 struct instruction *insn, 3433 struct symbol *func) 3434 { 3435 /* 3436 * We can't deal with indirect function calls at present; 3437 * assume they're instrumented. 3438 */ 3439 if (!func) { 3440 if (file->pv_ops) 3441 return pv_call_dest(file, insn); 3442 3443 return false; 3444 } 3445 3446 /* 3447 * If the symbol is from a noinstr section; we good. 3448 */ 3449 if (func->sec->noinstr) 3450 return true; 3451 3452 /* 3453 * If the symbol is a static_call trampoline, we can't tell. 3454 */ 3455 if (func->static_call_tramp) 3456 return true; 3457 3458 /* 3459 * The __ubsan_handle_*() calls are like WARN(), they only happen when 3460 * something 'BAD' happened. At the risk of taking the machine down, 3461 * let them proceed to get the message out. 3462 */ 3463 if (!strncmp(func->name, "__ubsan_handle_", 15)) 3464 return true; 3465 3466 return false; 3467 } 3468 3469 static int validate_call(struct objtool_file *file, 3470 struct instruction *insn, 3471 struct insn_state *state) 3472 { 3473 if (state->noinstr && state->instr <= 0 && 3474 !noinstr_call_dest(file, insn, insn_call_dest(insn))) { 3475 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn)); 3476 return 1; 3477 } 3478 3479 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) { 3480 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn)); 3481 return 1; 3482 } 3483 3484 if (state->df) { 3485 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn)); 3486 return 1; 3487 } 3488 3489 return 0; 3490 } 3491 3492 static int validate_sibling_call(struct objtool_file *file, 3493 struct instruction *insn, 3494 struct insn_state *state) 3495 { 3496 if (insn_func(insn) && has_modified_stack_frame(insn, state)) { 3497 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame"); 3498 return 1; 3499 } 3500 3501 return validate_call(file, insn, state); 3502 } 3503 3504 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 3505 { 3506 if (state->noinstr && state->instr > 0) { 3507 WARN_INSN(insn, "return with instrumentation enabled"); 3508 return 1; 3509 } 3510 3511 if (state->uaccess && !func_uaccess_safe(func)) { 3512 WARN_INSN(insn, "return with UACCESS enabled"); 3513 return 1; 3514 } 3515 3516 if (!state->uaccess && func_uaccess_safe(func)) { 3517 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function"); 3518 return 1; 3519 } 3520 3521 if (state->df) { 3522 WARN_INSN(insn, "return with DF set"); 3523 return 1; 3524 } 3525 3526 if (func && has_modified_stack_frame(insn, state)) { 3527 WARN_INSN(insn, "return with modified stack frame"); 3528 return 1; 3529 } 3530 3531 if (state->cfi.bp_scratch) { 3532 WARN_INSN(insn, "BP used as a scratch register"); 3533 return 1; 3534 } 3535 3536 return 0; 3537 } 3538 3539 static struct instruction *next_insn_to_validate(struct objtool_file *file, 3540 struct instruction *insn) 3541 { 3542 struct alt_group *alt_group = insn->alt_group; 3543 3544 /* 3545 * Simulate the fact that alternatives are patched in-place. When the 3546 * end of a replacement alt_group is reached, redirect objtool flow to 3547 * the end of the original alt_group. 3548 * 3549 * insn->alts->insn -> alt_group->first_insn 3550 * ... 3551 * alt_group->last_insn 3552 * [alt_group->nop] -> next(orig_group->last_insn) 3553 */ 3554 if (alt_group) { 3555 if (alt_group->nop) { 3556 /* ->nop implies ->orig_group */ 3557 if (insn == alt_group->last_insn) 3558 return alt_group->nop; 3559 if (insn == alt_group->nop) 3560 goto next_orig; 3561 } 3562 if (insn == alt_group->last_insn && alt_group->orig_group) 3563 goto next_orig; 3564 } 3565 3566 return next_insn_same_sec(file, insn); 3567 3568 next_orig: 3569 return next_insn_same_sec(file, alt_group->orig_group->last_insn); 3570 } 3571 3572 static bool skip_alt_group(struct instruction *insn) 3573 { 3574 struct instruction *alt_insn = insn->alts ? insn->alts->insn : NULL; 3575 3576 /* ANNOTATE_IGNORE_ALTERNATIVE */ 3577 if (insn->alt_group && insn->alt_group->ignore) 3578 return true; 3579 3580 /* 3581 * For NOP patched with CLAC/STAC, only follow the latter to avoid 3582 * impossible code paths combining patched CLAC with unpatched STAC 3583 * or vice versa. 3584 * 3585 * ANNOTATE_IGNORE_ALTERNATIVE could have been used here, but Linus 3586 * requested not to do that to avoid hurting .s file readability 3587 * around CLAC/STAC alternative sites. 3588 */ 3589 3590 if (!alt_insn) 3591 return false; 3592 3593 /* Don't override ASM_{CLAC,STAC}_UNSAFE */ 3594 if (alt_insn->alt_group && alt_insn->alt_group->ignore) 3595 return false; 3596 3597 return alt_insn->type == INSN_CLAC || alt_insn->type == INSN_STAC; 3598 } 3599 3600 static int checksum_debug_init(struct objtool_file *file) 3601 { 3602 char *dup, *s; 3603 3604 if (!opts.debug_checksum) 3605 return 0; 3606 3607 dup = strdup(opts.debug_checksum); 3608 if (!dup) { 3609 ERROR_GLIBC("strdup"); 3610 return -1; 3611 } 3612 3613 s = dup; 3614 while (*s) { 3615 struct symbol *func; 3616 char *comma; 3617 3618 comma = strchr(s, ','); 3619 if (comma) 3620 *comma = '\0'; 3621 3622 func = find_symbol_by_name(file->elf, s); 3623 if (!func || !is_func_sym(func)) 3624 WARN("--debug-checksum: can't find '%s'", s); 3625 else 3626 func->debug_checksum = 1; 3627 3628 if (!comma) 3629 break; 3630 3631 s = comma + 1; 3632 } 3633 3634 free(dup); 3635 return 0; 3636 } 3637 3638 static void checksum_update_insn(struct objtool_file *file, struct symbol *func, 3639 struct instruction *insn) 3640 { 3641 struct reloc *reloc = insn_reloc(file, insn); 3642 unsigned long offset; 3643 struct symbol *sym; 3644 3645 if (insn->fake) 3646 return; 3647 3648 checksum_update(func, insn, insn->sec->data->d_buf + insn->offset, insn->len); 3649 3650 if (!reloc) { 3651 struct symbol *call_dest = insn_call_dest(insn); 3652 3653 if (call_dest) 3654 checksum_update(func, insn, call_dest->demangled_name, 3655 strlen(call_dest->demangled_name)); 3656 return; 3657 } 3658 3659 sym = reloc->sym; 3660 offset = arch_insn_adjusted_addend(insn, reloc); 3661 3662 if (is_string_sec(sym->sec)) { 3663 char *str; 3664 3665 str = sym->sec->data->d_buf + sym->offset + offset; 3666 checksum_update(func, insn, str, strlen(str)); 3667 return; 3668 } 3669 3670 if (is_sec_sym(sym)) { 3671 sym = find_symbol_containing(reloc->sym->sec, offset); 3672 if (!sym) 3673 return; 3674 3675 offset -= sym->offset; 3676 } 3677 3678 checksum_update(func, insn, sym->demangled_name, strlen(sym->demangled_name)); 3679 checksum_update(func, insn, &offset, sizeof(offset)); 3680 } 3681 3682 /* 3683 * Follow the branch starting at the given instruction, and recursively follow 3684 * any other branches (jumps). Meanwhile, track the frame pointer state at 3685 * each instruction and validate all the rules described in 3686 * tools/objtool/Documentation/objtool.txt. 3687 */ 3688 static int validate_branch(struct objtool_file *file, struct symbol *func, 3689 struct instruction *insn, struct insn_state state) 3690 { 3691 struct alternative *alt; 3692 struct instruction *next_insn, *prev_insn = NULL; 3693 u8 visited; 3694 int ret; 3695 3696 if (func && func->ignore) 3697 return 0; 3698 3699 while (1) { 3700 next_insn = next_insn_to_validate(file, insn); 3701 3702 if (opts.checksum && func && insn->sec) 3703 checksum_update_insn(file, func, insn); 3704 3705 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { 3706 /* Ignore KCFI type preambles, which always fall through */ 3707 if (is_prefix_func(func)) 3708 return 0; 3709 3710 if (file->ignore_unreachables) 3711 return 0; 3712 3713 WARN("%s() falls through to next function %s()", 3714 func->name, insn_func(insn)->name); 3715 func->warned = 1; 3716 3717 return 1; 3718 } 3719 3720 visited = VISITED_BRANCH << state.uaccess; 3721 if (insn->visited & VISITED_BRANCH_MASK) { 3722 if (!insn->hint && !insn_cfi_match(insn, &state.cfi)) 3723 return 1; 3724 3725 if (insn->visited & visited) 3726 return 0; 3727 } else { 3728 nr_insns_visited++; 3729 } 3730 3731 if (state.noinstr) 3732 state.instr += insn->instr; 3733 3734 if (insn->hint) { 3735 if (insn->restore) { 3736 struct instruction *save_insn, *i; 3737 3738 i = insn; 3739 save_insn = NULL; 3740 3741 sym_for_each_insn_continue_reverse(file, func, i) { 3742 if (i->save) { 3743 save_insn = i; 3744 break; 3745 } 3746 } 3747 3748 if (!save_insn) { 3749 WARN_INSN(insn, "no corresponding CFI save for CFI restore"); 3750 return 1; 3751 } 3752 3753 if (!save_insn->visited) { 3754 /* 3755 * If the restore hint insn is at the 3756 * beginning of a basic block and was 3757 * branched to from elsewhere, and the 3758 * save insn hasn't been visited yet, 3759 * defer following this branch for now. 3760 * It will be seen later via the 3761 * straight-line path. 3762 */ 3763 if (!prev_insn) 3764 return 0; 3765 3766 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo"); 3767 return 1; 3768 } 3769 3770 insn->cfi = save_insn->cfi; 3771 nr_cfi_reused++; 3772 } 3773 3774 state.cfi = *insn->cfi; 3775 } else { 3776 /* XXX track if we actually changed state.cfi */ 3777 3778 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) { 3779 insn->cfi = prev_insn->cfi; 3780 nr_cfi_reused++; 3781 } else { 3782 insn->cfi = cfi_hash_find_or_add(&state.cfi); 3783 } 3784 } 3785 3786 insn->visited |= visited; 3787 3788 if (propagate_alt_cfi(file, insn)) 3789 return 1; 3790 3791 if (insn->alts) { 3792 for (alt = insn->alts; alt; alt = alt->next) { 3793 ret = validate_branch(file, func, alt->insn, state); 3794 if (ret) { 3795 BT_INSN(insn, "(alt)"); 3796 return ret; 3797 } 3798 } 3799 } 3800 3801 if (skip_alt_group(insn)) 3802 return 0; 3803 3804 if (handle_insn_ops(insn, next_insn, &state)) 3805 return 1; 3806 3807 switch (insn->type) { 3808 3809 case INSN_RETURN: 3810 return validate_return(func, insn, &state); 3811 3812 case INSN_CALL: 3813 case INSN_CALL_DYNAMIC: 3814 ret = validate_call(file, insn, &state); 3815 if (ret) 3816 return ret; 3817 3818 if (opts.stackval && func && !is_special_call(insn) && 3819 !has_valid_stack_frame(&state)) { 3820 WARN_INSN(insn, "call without frame pointer save/setup"); 3821 return 1; 3822 } 3823 3824 break; 3825 3826 case INSN_JUMP_CONDITIONAL: 3827 case INSN_JUMP_UNCONDITIONAL: 3828 if (is_sibling_call(insn)) { 3829 ret = validate_sibling_call(file, insn, &state); 3830 if (ret) 3831 return ret; 3832 3833 } else if (insn->jump_dest) { 3834 ret = validate_branch(file, func, 3835 insn->jump_dest, state); 3836 if (ret) { 3837 BT_INSN(insn, "(branch)"); 3838 return ret; 3839 } 3840 } 3841 3842 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3843 return 0; 3844 3845 break; 3846 3847 case INSN_JUMP_DYNAMIC: 3848 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3849 if (is_sibling_call(insn)) { 3850 ret = validate_sibling_call(file, insn, &state); 3851 if (ret) 3852 return ret; 3853 } 3854 3855 if (insn->type == INSN_JUMP_DYNAMIC) 3856 return 0; 3857 3858 break; 3859 3860 case INSN_SYSCALL: 3861 if (func && (!next_insn || !next_insn->hint)) { 3862 WARN_INSN(insn, "unsupported instruction in callable function"); 3863 return 1; 3864 } 3865 3866 break; 3867 3868 case INSN_SYSRET: 3869 if (func && (!next_insn || !next_insn->hint)) { 3870 WARN_INSN(insn, "unsupported instruction in callable function"); 3871 return 1; 3872 } 3873 3874 return 0; 3875 3876 case INSN_STAC: 3877 if (!opts.uaccess) 3878 break; 3879 3880 if (state.uaccess) { 3881 WARN_INSN(insn, "recursive UACCESS enable"); 3882 return 1; 3883 } 3884 3885 state.uaccess = true; 3886 break; 3887 3888 case INSN_CLAC: 3889 if (!opts.uaccess) 3890 break; 3891 3892 if (!state.uaccess && func) { 3893 WARN_INSN(insn, "redundant UACCESS disable"); 3894 return 1; 3895 } 3896 3897 if (func_uaccess_safe(func) && !state.uaccess_stack) { 3898 WARN_INSN(insn, "UACCESS-safe disables UACCESS"); 3899 return 1; 3900 } 3901 3902 state.uaccess = false; 3903 break; 3904 3905 case INSN_STD: 3906 if (state.df) { 3907 WARN_INSN(insn, "recursive STD"); 3908 return 1; 3909 } 3910 3911 state.df = true; 3912 break; 3913 3914 case INSN_CLD: 3915 if (!state.df && func) { 3916 WARN_INSN(insn, "redundant CLD"); 3917 return 1; 3918 } 3919 3920 state.df = false; 3921 break; 3922 3923 default: 3924 break; 3925 } 3926 3927 if (insn->dead_end) 3928 return 0; 3929 3930 if (!next_insn) { 3931 if (state.cfi.cfa.base == CFI_UNDEFINED) 3932 return 0; 3933 if (file->ignore_unreachables) 3934 return 0; 3935 3936 WARN("%s%sunexpected end of section %s", 3937 func ? func->name : "", func ? "(): " : "", 3938 insn->sec->name); 3939 return 1; 3940 } 3941 3942 prev_insn = insn; 3943 insn = next_insn; 3944 } 3945 3946 return 0; 3947 } 3948 3949 static int validate_unwind_hint(struct objtool_file *file, 3950 struct instruction *insn, 3951 struct insn_state *state) 3952 { 3953 if (insn->hint && !insn->visited) { 3954 struct symbol *func = insn_func(insn); 3955 int ret; 3956 3957 if (opts.checksum) 3958 checksum_init(func); 3959 3960 ret = validate_branch(file, func, insn, *state); 3961 if (ret) 3962 BT_INSN(insn, "<=== (hint)"); 3963 return ret; 3964 } 3965 3966 return 0; 3967 } 3968 3969 static int validate_unwind_hints(struct objtool_file *file, struct section *sec) 3970 { 3971 struct instruction *insn; 3972 struct insn_state state; 3973 int warnings = 0; 3974 3975 if (!file->hints) 3976 return 0; 3977 3978 init_insn_state(file, &state, sec); 3979 3980 if (sec) { 3981 sec_for_each_insn(file, sec, insn) 3982 warnings += validate_unwind_hint(file, insn, &state); 3983 } else { 3984 for_each_insn(file, insn) 3985 warnings += validate_unwind_hint(file, insn, &state); 3986 } 3987 3988 return warnings; 3989 } 3990 3991 /* 3992 * Validate rethunk entry constraint: must untrain RET before the first RET. 3993 * 3994 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes 3995 * before an actual RET instruction. 3996 */ 3997 static int validate_unret(struct objtool_file *file, struct instruction *insn) 3998 { 3999 struct instruction *next, *dest; 4000 int ret; 4001 4002 for (;;) { 4003 next = next_insn_to_validate(file, insn); 4004 4005 if (insn->visited & VISITED_UNRET) 4006 return 0; 4007 4008 insn->visited |= VISITED_UNRET; 4009 4010 if (insn->alts) { 4011 struct alternative *alt; 4012 for (alt = insn->alts; alt; alt = alt->next) { 4013 ret = validate_unret(file, alt->insn); 4014 if (ret) { 4015 BT_INSN(insn, "(alt)"); 4016 return ret; 4017 } 4018 } 4019 } 4020 4021 switch (insn->type) { 4022 4023 case INSN_CALL_DYNAMIC: 4024 case INSN_JUMP_DYNAMIC: 4025 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4026 WARN_INSN(insn, "early indirect call"); 4027 return 1; 4028 4029 case INSN_JUMP_UNCONDITIONAL: 4030 case INSN_JUMP_CONDITIONAL: 4031 if (!is_sibling_call(insn)) { 4032 if (!insn->jump_dest) { 4033 WARN_INSN(insn, "unresolved jump target after linking?!?"); 4034 return 1; 4035 } 4036 ret = validate_unret(file, insn->jump_dest); 4037 if (ret) { 4038 BT_INSN(insn, "(branch%s)", 4039 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : ""); 4040 return ret; 4041 } 4042 4043 if (insn->type == INSN_JUMP_UNCONDITIONAL) 4044 return 0; 4045 4046 break; 4047 } 4048 4049 /* fallthrough */ 4050 case INSN_CALL: 4051 dest = find_insn(file, insn_call_dest(insn)->sec, 4052 insn_call_dest(insn)->offset); 4053 if (!dest) { 4054 WARN("Unresolved function after linking!?: %s", 4055 insn_call_dest(insn)->name); 4056 return 1; 4057 } 4058 4059 ret = validate_unret(file, dest); 4060 if (ret) { 4061 BT_INSN(insn, "(call)"); 4062 return ret; 4063 } 4064 /* 4065 * If a call returns without error, it must have seen UNTRAIN_RET. 4066 * Therefore any non-error return is a success. 4067 */ 4068 return 0; 4069 4070 case INSN_RETURN: 4071 WARN_INSN(insn, "RET before UNTRAIN"); 4072 return 1; 4073 4074 case INSN_SYSCALL: 4075 break; 4076 4077 case INSN_SYSRET: 4078 return 0; 4079 4080 case INSN_NOP: 4081 if (insn->retpoline_safe) 4082 return 0; 4083 break; 4084 4085 default: 4086 break; 4087 } 4088 4089 if (insn->dead_end) 4090 return 0; 4091 4092 if (!next) { 4093 WARN_INSN(insn, "teh end!"); 4094 return 1; 4095 } 4096 insn = next; 4097 } 4098 4099 return 0; 4100 } 4101 4102 /* 4103 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter 4104 * VALIDATE_UNRET_END before RET. 4105 */ 4106 static int validate_unrets(struct objtool_file *file) 4107 { 4108 struct instruction *insn; 4109 int warnings = 0; 4110 4111 for_each_insn(file, insn) { 4112 if (!insn->unret) 4113 continue; 4114 4115 warnings += validate_unret(file, insn); 4116 } 4117 4118 return warnings; 4119 } 4120 4121 static int validate_retpoline(struct objtool_file *file) 4122 { 4123 struct instruction *insn; 4124 int warnings = 0; 4125 4126 for_each_insn(file, insn) { 4127 if (insn->type != INSN_JUMP_DYNAMIC && 4128 insn->type != INSN_CALL_DYNAMIC && 4129 insn->type != INSN_RETURN) 4130 continue; 4131 4132 if (insn->retpoline_safe) 4133 continue; 4134 4135 if (insn->sec->init) 4136 continue; 4137 4138 if (insn->type == INSN_RETURN) { 4139 if (opts.rethunk) { 4140 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build"); 4141 warnings++; 4142 } 4143 continue; 4144 } 4145 4146 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build", 4147 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); 4148 warnings++; 4149 } 4150 4151 if (!opts.cfi) 4152 return warnings; 4153 4154 /* 4155 * kCFI call sites look like: 4156 * 4157 * movl $(-0x12345678), %r10d 4158 * addl -4(%r11), %r10d 4159 * jz 1f 4160 * ud2 4161 * 1: cs call __x86_indirect_thunk_r11 4162 * 4163 * Verify all indirect calls are kCFI adorned by checking for the 4164 * UD2. Notably, doing __nocfi calls to regular (cfi) functions is 4165 * broken. 4166 */ 4167 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 4168 struct symbol *sym = insn->sym; 4169 4170 if (sym && (sym->type == STT_NOTYPE || 4171 sym->type == STT_FUNC) && !sym->nocfi) { 4172 struct instruction *prev = 4173 prev_insn_same_sym(file, insn); 4174 4175 if (!prev || prev->type != INSN_BUG) { 4176 WARN_INSN(insn, "no-cfi indirect call!"); 4177 warnings++; 4178 } 4179 } 4180 } 4181 4182 return warnings; 4183 } 4184 4185 static bool is_kasan_insn(struct instruction *insn) 4186 { 4187 return (insn->type == INSN_CALL && 4188 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return")); 4189 } 4190 4191 static bool is_ubsan_insn(struct instruction *insn) 4192 { 4193 return (insn->type == INSN_CALL && 4194 !strcmp(insn_call_dest(insn)->name, 4195 "__ubsan_handle_builtin_unreachable")); 4196 } 4197 4198 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) 4199 { 4200 struct symbol *func = insn_func(insn); 4201 struct instruction *prev_insn; 4202 int i; 4203 4204 if (insn->type == INSN_NOP || insn->type == INSN_TRAP || 4205 insn->hole || (func && func->ignore)) 4206 return true; 4207 4208 /* 4209 * Ignore alternative replacement instructions. This can happen 4210 * when a whitelisted function uses one of the ALTERNATIVE macros. 4211 */ 4212 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 4213 !strcmp(insn->sec->name, ".altinstr_aux")) 4214 return true; 4215 4216 if (!func) 4217 return false; 4218 4219 if (func->static_call_tramp) 4220 return true; 4221 4222 /* 4223 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees 4224 * __builtin_unreachable(). The BUG() macro has an unreachable() after 4225 * the UD2, which causes GCC's undefined trap logic to emit another UD2 4226 * (or occasionally a JMP to UD2). 4227 * 4228 * It may also insert a UD2 after calling a __noreturn function. 4229 */ 4230 prev_insn = prev_insn_same_sec(file, insn); 4231 if (prev_insn && prev_insn->dead_end && 4232 (insn->type == INSN_BUG || 4233 (insn->type == INSN_JUMP_UNCONDITIONAL && 4234 insn->jump_dest && insn->jump_dest->type == INSN_BUG))) 4235 return true; 4236 4237 /* 4238 * Check if this (or a subsequent) instruction is related to 4239 * CONFIG_UBSAN or CONFIG_KASAN. 4240 * 4241 * End the search at 5 instructions to avoid going into the weeds. 4242 */ 4243 for (i = 0; i < 5; i++) { 4244 4245 if (is_kasan_insn(insn) || is_ubsan_insn(insn)) 4246 return true; 4247 4248 if (insn->type == INSN_JUMP_UNCONDITIONAL) { 4249 if (insn->jump_dest && 4250 insn_func(insn->jump_dest) == func) { 4251 insn = insn->jump_dest; 4252 continue; 4253 } 4254 4255 break; 4256 } 4257 4258 if (insn->offset + insn->len >= func->offset + func->len) 4259 break; 4260 4261 insn = next_insn_same_sec(file, insn); 4262 } 4263 4264 return false; 4265 } 4266 4267 /* 4268 * For FineIBT or kCFI, a certain number of bytes preceding the function may be 4269 * NOPs. Those NOPs may be rewritten at runtime and executed, so give them a 4270 * proper function name: __pfx_<func>. 4271 * 4272 * The NOPs may not exist for the following cases: 4273 * 4274 * - compiler cloned functions (*.cold, *.part0, etc) 4275 * - asm functions created with inline asm or without SYM_FUNC_START() 4276 * 4277 * Also, the function may already have a prefix from a previous objtool run 4278 * (livepatch extracted functions, or manually running objtool multiple times). 4279 * 4280 * So return 0 if the NOPs are missing or the function already has a prefix 4281 * symbol. 4282 */ 4283 static int create_prefix_symbol(struct objtool_file *file, struct symbol *func) 4284 { 4285 struct instruction *insn, *prev; 4286 char name[SYM_NAME_LEN]; 4287 struct cfi_state *cfi; 4288 4289 if (!is_func_sym(func) || is_prefix_func(func) || 4290 func->cold || func->static_call_tramp) 4291 return 0; 4292 4293 if ((strlen(func->name) + sizeof("__pfx_") > SYM_NAME_LEN)) { 4294 WARN("%s: symbol name too long, can't create __pfx_ symbol", 4295 func->name); 4296 return 0; 4297 } 4298 4299 if (snprintf_check(name, SYM_NAME_LEN, "__pfx_%s", func->name)) 4300 return -1; 4301 4302 if (file->klp) { 4303 struct symbol *pfx; 4304 4305 pfx = find_symbol_by_offset(func->sec, func->offset - opts.prefix); 4306 if (pfx && is_prefix_func(pfx) && !strcmp(pfx->name, name)) 4307 return 0; 4308 } 4309 4310 insn = find_insn(file, func->sec, func->offset); 4311 if (!insn) { 4312 WARN("%s: can't find starting instruction", func->name); 4313 return -1; 4314 } 4315 4316 for (prev = prev_insn_same_sec(file, insn); 4317 prev; 4318 prev = prev_insn_same_sec(file, prev)) { 4319 u64 offset; 4320 4321 if (prev->type != INSN_NOP) 4322 return 0; 4323 4324 offset = func->offset - prev->offset; 4325 4326 if (offset > opts.prefix) 4327 return 0; 4328 4329 if (offset < opts.prefix) 4330 continue; 4331 4332 if (!elf_create_symbol(file->elf, name, func->sec, 4333 GELF_ST_BIND(func->sym.st_info), 4334 GELF_ST_TYPE(func->sym.st_info), 4335 prev->offset, opts.prefix)) 4336 return -1; 4337 4338 break; 4339 } 4340 4341 if (!prev) 4342 return 0; 4343 4344 if (!insn->cfi) { 4345 /* 4346 * This can happen if stack validation isn't enabled or the 4347 * function is annotated with STACK_FRAME_NON_STANDARD. 4348 */ 4349 return 0; 4350 } 4351 4352 /* Propagate insn->cfi to the prefix code */ 4353 cfi = cfi_hash_find_or_add(insn->cfi); 4354 for (; prev != insn; prev = next_insn_same_sec(file, prev)) 4355 prev->cfi = cfi; 4356 4357 return 0; 4358 } 4359 4360 static int create_prefix_symbols(struct objtool_file *file) 4361 { 4362 struct section *sec; 4363 struct symbol *func; 4364 4365 for_each_sec(file->elf, sec) { 4366 if (!is_text_sec(sec)) 4367 continue; 4368 4369 sec_for_each_sym(sec, func) { 4370 if (create_prefix_symbol(file, func)) 4371 return -1; 4372 } 4373 } 4374 4375 return 0; 4376 } 4377 4378 static int validate_symbol(struct objtool_file *file, struct section *sec, 4379 struct symbol *sym, struct insn_state *state) 4380 { 4381 struct instruction *insn; 4382 struct symbol *func; 4383 int ret; 4384 4385 if (!sym->len) { 4386 WARN("%s() is missing an ELF size annotation", sym->name); 4387 return 1; 4388 } 4389 4390 if (sym->pfunc != sym || sym->alias != sym) 4391 return 0; 4392 4393 insn = find_insn(file, sec, sym->offset); 4394 if (!insn || insn->visited) 4395 return 0; 4396 4397 if (opts.uaccess) 4398 state->uaccess = sym->uaccess_safe; 4399 4400 func = insn_func(insn); 4401 4402 if (opts.checksum) 4403 checksum_init(func); 4404 4405 ret = validate_branch(file, func, insn, *state); 4406 if (ret) 4407 BT_INSN(insn, "<=== (sym)"); 4408 4409 if (opts.checksum) 4410 checksum_finish(func); 4411 4412 return ret; 4413 } 4414 4415 static int validate_section(struct objtool_file *file, struct section *sec) 4416 { 4417 struct insn_state state; 4418 struct symbol *func; 4419 int warnings = 0; 4420 4421 sec_for_each_sym(sec, func) { 4422 if (!is_func_sym(func)) 4423 continue; 4424 4425 init_insn_state(file, &state, sec); 4426 set_func_state(&state.cfi); 4427 4428 warnings += validate_symbol(file, sec, func, &state); 4429 } 4430 4431 return warnings; 4432 } 4433 4434 static int validate_noinstr_sections(struct objtool_file *file) 4435 { 4436 struct section *sec; 4437 int warnings = 0; 4438 4439 sec = find_section_by_name(file->elf, ".noinstr.text"); 4440 if (sec) { 4441 warnings += validate_section(file, sec); 4442 warnings += validate_unwind_hints(file, sec); 4443 } 4444 4445 sec = find_section_by_name(file->elf, ".entry.text"); 4446 if (sec) { 4447 warnings += validate_section(file, sec); 4448 warnings += validate_unwind_hints(file, sec); 4449 } 4450 4451 sec = find_section_by_name(file->elf, ".cpuidle.text"); 4452 if (sec) { 4453 warnings += validate_section(file, sec); 4454 warnings += validate_unwind_hints(file, sec); 4455 } 4456 4457 return warnings; 4458 } 4459 4460 static int validate_functions(struct objtool_file *file) 4461 { 4462 struct section *sec; 4463 int warnings = 0; 4464 4465 for_each_sec(file->elf, sec) { 4466 if (!is_text_sec(sec)) 4467 continue; 4468 4469 warnings += validate_section(file, sec); 4470 } 4471 4472 return warnings; 4473 } 4474 4475 static void mark_endbr_used(struct instruction *insn) 4476 { 4477 if (!list_empty(&insn->call_node)) 4478 list_del_init(&insn->call_node); 4479 } 4480 4481 static bool noendbr_range(struct objtool_file *file, struct instruction *insn) 4482 { 4483 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1); 4484 struct instruction *first; 4485 4486 if (!sym) 4487 return false; 4488 4489 first = find_insn(file, sym->sec, sym->offset); 4490 if (!first) 4491 return false; 4492 4493 if (first->type != INSN_ENDBR && !first->noendbr) 4494 return false; 4495 4496 return insn->offset == sym->offset + sym->len; 4497 } 4498 4499 static int __validate_ibt_insn(struct objtool_file *file, struct instruction *insn, 4500 struct instruction *dest) 4501 { 4502 if (dest->type == INSN_ENDBR) { 4503 mark_endbr_used(dest); 4504 return 0; 4505 } 4506 4507 if (insn_func(dest) && insn_func(insn) && 4508 insn_func(dest)->pfunc == insn_func(insn)->pfunc) { 4509 /* 4510 * Anything from->to self is either _THIS_IP_ or 4511 * IRET-to-self. 4512 * 4513 * There is no sane way to annotate _THIS_IP_ since the 4514 * compiler treats the relocation as a constant and is 4515 * happy to fold in offsets, skewing any annotation we 4516 * do, leading to vast amounts of false-positives. 4517 * 4518 * There's also compiler generated _THIS_IP_ through 4519 * KCOV and such which we have no hope of annotating. 4520 * 4521 * As such, blanket accept self-references without 4522 * issue. 4523 */ 4524 return 0; 4525 } 4526 4527 /* 4528 * Accept anything ANNOTATE_NOENDBR. 4529 */ 4530 if (dest->noendbr) 4531 return 0; 4532 4533 /* 4534 * Accept if this is the instruction after a symbol 4535 * that is (no)endbr -- typical code-range usage. 4536 */ 4537 if (noendbr_range(file, dest)) 4538 return 0; 4539 4540 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4541 return 1; 4542 } 4543 4544 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn) 4545 { 4546 struct instruction *dest; 4547 struct reloc *reloc; 4548 unsigned long off; 4549 int warnings = 0; 4550 4551 /* 4552 * Looking for function pointer load relocations. Ignore 4553 * direct/indirect branches: 4554 */ 4555 switch (insn->type) { 4556 4557 case INSN_CALL: 4558 case INSN_CALL_DYNAMIC: 4559 case INSN_JUMP_CONDITIONAL: 4560 case INSN_JUMP_UNCONDITIONAL: 4561 case INSN_JUMP_DYNAMIC: 4562 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4563 case INSN_RETURN: 4564 case INSN_NOP: 4565 return 0; 4566 4567 case INSN_LEA_RIP: 4568 if (!insn_reloc(file, insn)) { 4569 /* local function pointer reference without reloc */ 4570 4571 off = arch_jump_destination(insn); 4572 4573 dest = find_insn(file, insn->sec, off); 4574 if (!dest) { 4575 WARN_INSN(insn, "corrupt function pointer reference"); 4576 return 1; 4577 } 4578 4579 return __validate_ibt_insn(file, insn, dest); 4580 } 4581 break; 4582 4583 default: 4584 break; 4585 } 4586 4587 for (reloc = insn_reloc(file, insn); 4588 reloc; 4589 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 4590 reloc_offset(reloc) + 1, 4591 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) { 4592 4593 off = reloc->sym->offset + arch_insn_adjusted_addend(insn, reloc); 4594 4595 dest = find_insn(file, reloc->sym->sec, off); 4596 if (!dest) 4597 continue; 4598 4599 warnings += __validate_ibt_insn(file, insn, dest); 4600 } 4601 4602 return warnings; 4603 } 4604 4605 static int validate_ibt_data_reloc(struct objtool_file *file, 4606 struct reloc *reloc) 4607 { 4608 struct instruction *dest; 4609 4610 dest = find_insn(file, reloc->sym->sec, 4611 reloc->sym->offset + reloc_addend(reloc)); 4612 if (!dest) 4613 return 0; 4614 4615 if (dest->type == INSN_ENDBR) { 4616 mark_endbr_used(dest); 4617 return 0; 4618 } 4619 4620 if (dest->noendbr) 4621 return 0; 4622 4623 WARN_FUNC(reloc->sec->base, reloc_offset(reloc), 4624 "data relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4625 4626 return 1; 4627 } 4628 4629 /* 4630 * Validate IBT rules and remove used ENDBR instructions from the seal list. 4631 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with 4632 * NOPs) later, in create_ibt_endbr_seal_sections(). 4633 */ 4634 static int validate_ibt(struct objtool_file *file) 4635 { 4636 struct section *sec; 4637 struct reloc *reloc; 4638 struct instruction *insn; 4639 int warnings = 0; 4640 4641 for_each_insn(file, insn) 4642 warnings += validate_ibt_insn(file, insn); 4643 4644 for_each_sec(file->elf, sec) { 4645 4646 /* Already done by validate_ibt_insn() */ 4647 if (is_text_sec(sec)) 4648 continue; 4649 4650 if (!sec->rsec) 4651 continue; 4652 4653 /* 4654 * These sections can reference text addresses, but not with 4655 * the intent to indirect branch to them. 4656 */ 4657 if ((!strncmp(sec->name, ".discard", 8) && 4658 strcmp(sec->name, ".discard.ibt_endbr_noseal")) || 4659 !strncmp(sec->name, ".debug", 6) || 4660 !strcmp(sec->name, ".altinstructions") || 4661 !strcmp(sec->name, ".ibt_endbr_seal") || 4662 !strcmp(sec->name, ".kcfi_traps") || 4663 !strcmp(sec->name, ".orc_unwind_ip") || 4664 !strcmp(sec->name, ".retpoline_sites") || 4665 !strcmp(sec->name, ".smp_locks") || 4666 !strcmp(sec->name, ".static_call_sites") || 4667 !strcmp(sec->name, "_error_injection_whitelist") || 4668 !strcmp(sec->name, "_kprobe_blacklist") || 4669 !strcmp(sec->name, "__bug_table") || 4670 !strcmp(sec->name, "__ex_table") || 4671 !strcmp(sec->name, "__jump_table") || 4672 !strcmp(sec->name, "__klp_funcs") || 4673 !strcmp(sec->name, "__mcount_loc") || 4674 !strcmp(sec->name, ".llvm.call-graph-profile") || 4675 !strcmp(sec->name, ".llvm_bb_addr_map") || 4676 !strcmp(sec->name, "__tracepoints") || 4677 !strcmp(sec->name, "__patchable_function_entries")) 4678 continue; 4679 4680 for_each_reloc(sec->rsec, reloc) 4681 warnings += validate_ibt_data_reloc(file, reloc); 4682 } 4683 4684 return warnings; 4685 } 4686 4687 static int validate_sls(struct objtool_file *file) 4688 { 4689 struct instruction *insn, *next_insn; 4690 int warnings = 0; 4691 4692 for_each_insn(file, insn) { 4693 next_insn = next_insn_same_sec(file, insn); 4694 4695 if (insn->retpoline_safe) 4696 continue; 4697 4698 switch (insn->type) { 4699 case INSN_RETURN: 4700 if (!next_insn || next_insn->type != INSN_TRAP) { 4701 WARN_INSN(insn, "missing int3 after ret"); 4702 warnings++; 4703 } 4704 4705 break; 4706 case INSN_JUMP_DYNAMIC: 4707 if (!next_insn || next_insn->type != INSN_TRAP) { 4708 WARN_INSN(insn, "missing int3 after indirect jump"); 4709 warnings++; 4710 } 4711 break; 4712 default: 4713 break; 4714 } 4715 } 4716 4717 return warnings; 4718 } 4719 4720 static int validate_reachable_instructions(struct objtool_file *file) 4721 { 4722 struct instruction *insn, *prev_insn; 4723 struct symbol *call_dest; 4724 int warnings = 0; 4725 4726 if (file->ignore_unreachables) 4727 return 0; 4728 4729 for_each_insn(file, insn) { 4730 if (insn->visited || ignore_unreachable_insn(file, insn)) 4731 continue; 4732 4733 prev_insn = prev_insn_same_sec(file, insn); 4734 if (prev_insn && prev_insn->dead_end) { 4735 call_dest = insn_call_dest(prev_insn); 4736 if (call_dest) { 4737 WARN_INSN(insn, "%s() missing __noreturn in .c/.h or NORETURN() in noreturns.h", 4738 call_dest->name); 4739 warnings++; 4740 continue; 4741 } 4742 } 4743 4744 WARN_INSN(insn, "unreachable instruction"); 4745 warnings++; 4746 } 4747 4748 return warnings; 4749 } 4750 4751 /* 'funcs' is a space-separated list of function names */ 4752 static void disas_funcs(const char *funcs) 4753 { 4754 const char *objdump_str, *cross_compile; 4755 int size, ret; 4756 char *cmd; 4757 4758 cross_compile = getenv("CROSS_COMPILE"); 4759 if (!cross_compile) 4760 cross_compile = ""; 4761 4762 objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '" 4763 "BEGIN { split(_funcs, funcs); }" 4764 "/^$/ { func_match = 0; }" 4765 "/<.*>:/ { " 4766 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);" 4767 "for (i in funcs) {" 4768 "if (funcs[i] == f) {" 4769 "func_match = 1;" 4770 "base = strtonum(\"0x\" $1);" 4771 "break;" 4772 "}" 4773 "}" 4774 "}" 4775 "{" 4776 "if (func_match) {" 4777 "addr = strtonum(\"0x\" $1);" 4778 "printf(\"%%04x \", addr - base);" 4779 "print;" 4780 "}" 4781 "}' 1>&2"; 4782 4783 /* fake snprintf() to calculate the size */ 4784 size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1; 4785 if (size <= 0) { 4786 WARN("objdump string size calculation failed"); 4787 return; 4788 } 4789 4790 cmd = malloc(size); 4791 4792 /* real snprintf() */ 4793 snprintf(cmd, size, objdump_str, cross_compile, objname, funcs); 4794 ret = system(cmd); 4795 if (ret) { 4796 WARN("disassembly failed: %d", ret); 4797 return; 4798 } 4799 } 4800 4801 static void disas_warned_funcs(struct objtool_file *file) 4802 { 4803 struct symbol *sym; 4804 char *funcs = NULL, *tmp; 4805 4806 for_each_sym(file->elf, sym) { 4807 if (sym->warned) { 4808 if (!funcs) { 4809 funcs = malloc(strlen(sym->name) + 1); 4810 if (!funcs) { 4811 ERROR_GLIBC("malloc"); 4812 return; 4813 } 4814 strcpy(funcs, sym->name); 4815 } else { 4816 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2); 4817 if (!tmp) { 4818 ERROR_GLIBC("malloc"); 4819 return; 4820 } 4821 sprintf(tmp, "%s %s", funcs, sym->name); 4822 free(funcs); 4823 funcs = tmp; 4824 } 4825 } 4826 } 4827 4828 if (funcs) 4829 disas_funcs(funcs); 4830 } 4831 4832 __weak bool arch_absolute_reloc(struct elf *elf, struct reloc *reloc) 4833 { 4834 unsigned int type = reloc_type(reloc); 4835 size_t sz = elf_addr_size(elf); 4836 4837 return (sz == 8) ? (type == R_ABS64) : (type == R_ABS32); 4838 } 4839 4840 static int check_abs_references(struct objtool_file *file) 4841 { 4842 struct section *sec; 4843 struct reloc *reloc; 4844 int ret = 0; 4845 4846 for_each_sec(file->elf, sec) { 4847 /* absolute references in non-loadable sections are fine */ 4848 if (!(sec->sh.sh_flags & SHF_ALLOC)) 4849 continue; 4850 4851 /* section must have an associated .rela section */ 4852 if (!sec->rsec) 4853 continue; 4854 4855 /* 4856 * Special case for compiler generated metadata that is not 4857 * consumed until after boot. 4858 */ 4859 if (!strcmp(sec->name, "__patchable_function_entries")) 4860 continue; 4861 4862 for_each_reloc(sec->rsec, reloc) { 4863 if (arch_absolute_reloc(file->elf, reloc)) { 4864 WARN("section %s has absolute relocation at offset 0x%lx", 4865 sec->name, reloc_offset(reloc)); 4866 ret++; 4867 } 4868 } 4869 } 4870 return ret; 4871 } 4872 4873 struct insn_chunk { 4874 void *addr; 4875 struct insn_chunk *next; 4876 }; 4877 4878 /* 4879 * Reduce peak RSS usage by freeing insns memory before writing the ELF file, 4880 * which can trigger more allocations for .debug_* sections whose data hasn't 4881 * been read yet. 4882 */ 4883 static void free_insns(struct objtool_file *file) 4884 { 4885 struct instruction *insn; 4886 struct insn_chunk *chunks = NULL, *chunk; 4887 4888 for_each_insn(file, insn) { 4889 if (!insn->idx) { 4890 chunk = malloc(sizeof(*chunk)); 4891 chunk->addr = insn; 4892 chunk->next = chunks; 4893 chunks = chunk; 4894 } 4895 } 4896 4897 for (chunk = chunks; chunk; chunk = chunk->next) 4898 free(chunk->addr); 4899 } 4900 4901 int check(struct objtool_file *file) 4902 { 4903 int ret = 0, warnings = 0; 4904 4905 arch_initial_func_cfi_state(&initial_func_cfi); 4906 init_cfi_state(&init_cfi); 4907 init_cfi_state(&func_cfi); 4908 set_func_state(&func_cfi); 4909 init_cfi_state(&force_undefined_cfi); 4910 force_undefined_cfi.force_undefined = true; 4911 4912 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) { 4913 ret = -1; 4914 goto out; 4915 } 4916 4917 cfi_hash_add(&init_cfi); 4918 cfi_hash_add(&func_cfi); 4919 4920 ret = checksum_debug_init(file); 4921 if (ret) 4922 goto out; 4923 4924 ret = decode_sections(file); 4925 if (ret) 4926 goto out; 4927 4928 if (!nr_insns) 4929 goto out; 4930 4931 if (opts.retpoline) 4932 warnings += validate_retpoline(file); 4933 4934 if (validate_branch_enabled()) { 4935 int w = 0; 4936 4937 w += validate_functions(file); 4938 w += validate_unwind_hints(file, NULL); 4939 if (!w) 4940 w += validate_reachable_instructions(file); 4941 4942 warnings += w; 4943 4944 } else if (opts.noinstr) { 4945 warnings += validate_noinstr_sections(file); 4946 } 4947 4948 if (opts.unret) { 4949 /* 4950 * Must be after validate_branch() and friends, it plays 4951 * further games with insn->visited. 4952 */ 4953 warnings += validate_unrets(file); 4954 } 4955 4956 if (opts.ibt) 4957 warnings += validate_ibt(file); 4958 4959 if (opts.sls) 4960 warnings += validate_sls(file); 4961 4962 if (opts.static_call) { 4963 ret = create_static_call_sections(file); 4964 if (ret) 4965 goto out; 4966 } 4967 4968 if (opts.retpoline) { 4969 ret = create_retpoline_sites_sections(file); 4970 if (ret) 4971 goto out; 4972 } 4973 4974 if (opts.cfi) { 4975 ret = create_cfi_sections(file); 4976 if (ret) 4977 goto out; 4978 } 4979 4980 if (opts.rethunk) { 4981 ret = create_return_sites_sections(file); 4982 if (ret) 4983 goto out; 4984 4985 if (opts.hack_skylake) { 4986 ret = create_direct_call_sections(file); 4987 if (ret) 4988 goto out; 4989 } 4990 } 4991 4992 if (opts.mcount) { 4993 ret = create_mcount_loc_sections(file); 4994 if (ret) 4995 goto out; 4996 } 4997 4998 if (opts.prefix) { 4999 ret = create_prefix_symbols(file); 5000 if (ret) 5001 goto out; 5002 } 5003 5004 if (opts.ibt) { 5005 ret = create_ibt_endbr_seal_sections(file); 5006 if (ret) 5007 goto out; 5008 } 5009 5010 if (opts.noabs) 5011 warnings += check_abs_references(file); 5012 5013 if (opts.checksum) { 5014 ret = create_sym_checksum_section(file); 5015 if (ret) 5016 goto out; 5017 } 5018 5019 if (opts.orc && nr_insns) { 5020 ret = orc_create(file); 5021 if (ret) 5022 goto out; 5023 } 5024 5025 free_insns(file); 5026 5027 if (opts.stats) { 5028 printf("nr_insns_visited: %ld\n", nr_insns_visited); 5029 printf("nr_cfi: %ld\n", nr_cfi); 5030 printf("nr_cfi_reused: %ld\n", nr_cfi_reused); 5031 printf("nr_cfi_cache: %ld\n", nr_cfi_cache); 5032 } 5033 5034 out: 5035 if (!ret && !warnings) 5036 return 0; 5037 5038 if (opts.werror && warnings) 5039 ret = 1; 5040 5041 if (opts.verbose) { 5042 if (opts.werror && warnings) 5043 WARN("%d warning(s) upgraded to errors", warnings); 5044 disas_warned_funcs(file); 5045 } 5046 5047 if (opts.backup && make_backup()) 5048 return 1; 5049 5050 return ret; 5051 } 5052