1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6 #define _GNU_SOURCE /* memmem() */ 7 #include <fnmatch.h> 8 #include <string.h> 9 #include <stdlib.h> 10 #include <inttypes.h> 11 #include <sys/mman.h> 12 13 #include <objtool/builtin.h> 14 #include <objtool/cfi.h> 15 #include <objtool/arch.h> 16 #include <objtool/disas.h> 17 #include <objtool/check.h> 18 #include <objtool/special.h> 19 #include <objtool/trace.h> 20 #include <objtool/warn.h> 21 #include <objtool/checksum.h> 22 #include <objtool/util.h> 23 24 #include <linux/objtool_types.h> 25 #include <linux/hashtable.h> 26 #include <linux/kernel.h> 27 #include <linux/static_call_types.h> 28 #include <linux/string.h> 29 30 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 31 32 static struct cfi_init_state initial_func_cfi; 33 static struct cfi_state init_cfi; 34 static struct cfi_state func_cfi; 35 static struct cfi_state force_undefined_cfi; 36 37 struct disas_context *objtool_disas_ctx; 38 39 size_t sym_name_max_len; 40 41 struct instruction *find_insn(struct objtool_file *file, 42 struct section *sec, unsigned long offset) 43 { 44 struct instruction *insn; 45 46 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) { 47 if (insn->sec == sec && insn->offset == offset) 48 return insn; 49 } 50 51 return NULL; 52 } 53 54 struct instruction *next_insn_same_sec(struct objtool_file *file, 55 struct instruction *insn) 56 { 57 if (insn->idx == INSN_CHUNK_MAX) 58 return find_insn(file, insn->sec, insn->offset + insn->len); 59 60 insn++; 61 if (!insn->len) 62 return NULL; 63 64 return insn; 65 } 66 67 struct instruction *next_insn_same_func(struct objtool_file *file, 68 struct instruction *insn) 69 { 70 struct instruction *next = next_insn_same_sec(file, insn); 71 struct symbol *func = insn_func(insn); 72 73 if (!func) 74 return NULL; 75 76 if (next && insn_func(next) == func) 77 return next; 78 79 /* Check if we're already in the subfunction: */ 80 if (func == func->cfunc) 81 return NULL; 82 83 /* Move to the subfunction: */ 84 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 85 } 86 87 static struct instruction *prev_insn_same_sec(struct objtool_file *file, 88 struct instruction *insn) 89 { 90 if (insn->idx == 0) { 91 if (insn->prev_len) 92 return find_insn(file, insn->sec, insn->offset - insn->prev_len); 93 return NULL; 94 } 95 96 return insn - 1; 97 } 98 99 static struct instruction *prev_insn_same_sym(struct objtool_file *file, 100 struct instruction *insn) 101 { 102 struct instruction *prev = prev_insn_same_sec(file, insn); 103 104 if (prev && insn_func(prev) == insn_func(insn)) 105 return prev; 106 107 return NULL; 108 } 109 110 #define for_each_insn(file, insn) \ 111 for (struct section *__sec, *__fake = (struct section *)1; \ 112 __fake; __fake = NULL) \ 113 for_each_sec(file->elf, __sec) \ 114 sec_for_each_insn(file, __sec, insn) 115 116 117 #define sym_for_each_insn(file, sym, insn) \ 118 for (insn = find_insn(file, sym->sec, sym->offset); \ 119 insn && insn->offset < sym->offset + sym->len; \ 120 insn = next_insn_same_sec(file, insn)) 121 122 #define sym_for_each_insn_continue_reverse(file, sym, insn) \ 123 for (insn = prev_insn_same_sec(file, insn); \ 124 insn && insn->offset >= sym->offset; \ 125 insn = prev_insn_same_sec(file, insn)) 126 127 #define sec_for_each_insn_from(file, insn) \ 128 for (; insn; insn = next_insn_same_sec(file, insn)) 129 130 #define sec_for_each_insn_continue(file, insn) \ 131 for (insn = next_insn_same_sec(file, insn); insn; \ 132 insn = next_insn_same_sec(file, insn)) 133 134 static inline struct reloc *insn_jump_table(struct instruction *insn) 135 { 136 if (insn->type == INSN_JUMP_DYNAMIC || 137 insn->type == INSN_CALL_DYNAMIC) 138 return insn->_jump_table; 139 140 return NULL; 141 } 142 143 static inline unsigned long insn_jump_table_size(struct instruction *insn) 144 { 145 if (insn->type == INSN_JUMP_DYNAMIC || 146 insn->type == INSN_CALL_DYNAMIC) 147 return insn->_jump_table_size; 148 149 return 0; 150 } 151 152 static bool is_jump_table_jump(struct instruction *insn) 153 { 154 struct alt_group *alt_group = insn->alt_group; 155 156 if (insn_jump_table(insn)) 157 return true; 158 159 /* Retpoline alternative for a jump table? */ 160 return alt_group && alt_group->orig_group && 161 insn_jump_table(alt_group->orig_group->first_insn); 162 } 163 164 static bool is_sibling_call(struct instruction *insn) 165 { 166 /* 167 * Assume only STT_FUNC calls have jump-tables. 168 */ 169 if (insn_func(insn)) { 170 /* An indirect jump is either a sibling call or a jump to a table. */ 171 if (insn->type == INSN_JUMP_DYNAMIC) 172 return !is_jump_table_jump(insn); 173 } 174 175 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */ 176 return (is_static_jump(insn) && insn_call_dest(insn)); 177 } 178 179 /* 180 * Checks if a function is a Rust "noreturn" one. 181 */ 182 static bool is_rust_noreturn(const struct symbol *func) 183 { 184 /* 185 * If it does not start with "_R", then it is not a Rust symbol. 186 */ 187 if (strncmp(func->name, "_R", 2)) 188 return false; 189 190 /* 191 * These are just heuristics -- we do not control the precise symbol 192 * name, due to the crate disambiguators (which depend on the compiler) 193 * as well as changes to the source code itself between versions (since 194 * these come from the Rust standard library). 195 */ 196 return str_ends_with(func->name, "_4core3num20from_str_radix_panic") || 197 str_ends_with(func->name, "_4core3num22from_ascii_radix_panic") || 198 str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || 199 str_ends_with(func->name, "_4core6option13expect_failed") || 200 str_ends_with(func->name, "_4core6option13unwrap_failed") || 201 str_ends_with(func->name, "_4core6result13unwrap_failed") || 202 str_ends_with(func->name, "_4core9panicking5panic") || 203 str_ends_with(func->name, "_4core9panicking9panic_fmt") || 204 str_ends_with(func->name, "_4core9panicking14panic_explicit") || 205 str_ends_with(func->name, "_4core9panicking14panic_nounwind") || 206 str_ends_with(func->name, "_4core9panicking18panic_bounds_check") || 207 str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt") || 208 str_ends_with(func->name, "_4core9panicking19assert_failed_inner") || 209 str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference") || 210 str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") || 211 str_ends_with(func->name, "_7___rustc17rust_begin_unwind") || 212 strstr(func->name, "_4core9panicking13assert_failed") || 213 strstr(func->name, "_4core9panicking11panic_const24panic_const_") || 214 (strstr(func->name, "_4core5slice5index") && 215 strstr(func->name, "slice_") && 216 str_ends_with(func->name, "_fail")); 217 } 218 219 /* 220 * This checks to see if the given function is a "noreturn" function. 221 * 222 * For global functions which are outside the scope of this object file, we 223 * have to keep a manual list of them. 224 * 225 * For local functions, we have to detect them manually by simply looking for 226 * the lack of a return instruction. 227 */ 228 static bool __dead_end_function(struct objtool_file *file, struct symbol *func, 229 int recursion) 230 { 231 int i; 232 struct instruction *insn; 233 bool empty = true; 234 235 #define NORETURN(func) __stringify(func), 236 static const char * const global_noreturns[] = { 237 #include "noreturns.h" 238 }; 239 #undef NORETURN 240 241 if (!func) 242 return false; 243 244 if (!is_local_sym(func)) { 245 if (is_rust_noreturn(func)) 246 return true; 247 248 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) 249 if (!strcmp(func->name, global_noreturns[i])) 250 return true; 251 } 252 253 if (is_weak_sym(func)) 254 return false; 255 256 if (!func->len) 257 return false; 258 259 insn = find_insn(file, func->sec, func->offset); 260 if (!insn || !insn_func(insn)) 261 return false; 262 263 func_for_each_insn(file, func, insn) { 264 empty = false; 265 266 if (insn->type == INSN_RETURN) 267 return false; 268 } 269 270 if (empty) 271 return false; 272 273 /* 274 * A function can have a sibling call instead of a return. In that 275 * case, the function's dead-end status depends on whether the target 276 * of the sibling call returns. 277 */ 278 func_for_each_insn(file, func, insn) { 279 if (is_sibling_call(insn)) { 280 struct instruction *dest = insn->jump_dest; 281 282 if (!dest) 283 /* sibling call to another file */ 284 return false; 285 286 /* local sibling call */ 287 if (recursion == 5) { 288 /* 289 * Infinite recursion: two functions have 290 * sibling calls to each other. This is a very 291 * rare case. It means they aren't dead ends. 292 */ 293 return false; 294 } 295 296 return __dead_end_function(file, insn_func(dest), recursion+1); 297 } 298 } 299 300 return true; 301 } 302 303 static bool dead_end_function(struct objtool_file *file, struct symbol *func) 304 { 305 return __dead_end_function(file, func, 0); 306 } 307 308 static void init_cfi_state(struct cfi_state *cfi) 309 { 310 int i; 311 312 for (i = 0; i < CFI_NUM_REGS; i++) { 313 cfi->regs[i].base = CFI_UNDEFINED; 314 cfi->vals[i].base = CFI_UNDEFINED; 315 } 316 cfi->cfa.base = CFI_UNDEFINED; 317 cfi->drap_reg = CFI_UNDEFINED; 318 cfi->drap_offset = -1; 319 } 320 321 static void init_insn_state(struct objtool_file *file, struct insn_state *state, 322 struct section *sec) 323 { 324 memset(state, 0, sizeof(*state)); 325 init_cfi_state(&state->cfi); 326 327 if (opts.noinstr && sec) 328 state->noinstr = sec->noinstr; 329 } 330 331 static struct cfi_state *cfi_alloc(void) 332 { 333 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state)); 334 if (!cfi) { 335 ERROR_GLIBC("calloc"); 336 exit(1); 337 } 338 nr_cfi++; 339 return cfi; 340 } 341 342 static int cfi_bits; 343 static struct hlist_head *cfi_hash; 344 345 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2) 346 { 347 return memcmp((void *)cfi1 + sizeof(cfi1->hash), 348 (void *)cfi2 + sizeof(cfi2->hash), 349 sizeof(struct cfi_state) - sizeof(struct hlist_node)); 350 } 351 352 static inline u32 cfi_key(struct cfi_state *cfi) 353 { 354 return jhash((void *)cfi + sizeof(cfi->hash), 355 sizeof(*cfi) - sizeof(cfi->hash), 0); 356 } 357 358 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi) 359 { 360 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 361 struct cfi_state *obj; 362 363 hlist_for_each_entry(obj, head, hash) { 364 if (!cficmp(cfi, obj)) { 365 nr_cfi_cache++; 366 return obj; 367 } 368 } 369 370 obj = cfi_alloc(); 371 *obj = *cfi; 372 hlist_add_head(&obj->hash, head); 373 374 return obj; 375 } 376 377 static void cfi_hash_add(struct cfi_state *cfi) 378 { 379 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 380 381 hlist_add_head(&cfi->hash, head); 382 } 383 384 static void *cfi_hash_alloc(unsigned long size) 385 { 386 cfi_bits = max(10, ilog2(size)); 387 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits, 388 PROT_READ|PROT_WRITE, 389 MAP_PRIVATE|MAP_ANON, -1, 0); 390 if (cfi_hash == (void *)-1L) { 391 ERROR_GLIBC("mmap fail cfi_hash"); 392 cfi_hash = NULL; 393 } else if (opts.stats) { 394 printf("cfi_bits: %d\n", cfi_bits); 395 } 396 397 return cfi_hash; 398 } 399 400 static unsigned long nr_insns; 401 static unsigned long nr_insns_visited; 402 403 /* 404 * Call the arch-specific instruction decoder for all the instructions and add 405 * them to the global instruction list. 406 */ 407 static int decode_instructions(struct objtool_file *file) 408 { 409 struct section *sec; 410 struct symbol *func; 411 unsigned long offset; 412 struct instruction *insn; 413 414 for_each_sec(file->elf, sec) { 415 struct instruction *insns = NULL; 416 u8 prev_len = 0; 417 u8 idx = 0; 418 419 if (!is_text_sec(sec)) 420 continue; 421 422 if (strcmp(sec->name, ".altinstr_replacement") && 423 strcmp(sec->name, ".altinstr_aux") && 424 strncmp(sec->name, ".discard.", 9)) 425 sec->text = true; 426 427 if (!strcmp(sec->name, ".noinstr.text") || 428 !strcmp(sec->name, ".entry.text") || 429 !strcmp(sec->name, ".cpuidle.text") || 430 !strncmp(sec->name, ".text..__x86.", 13)) 431 sec->noinstr = true; 432 433 /* 434 * .init.text code is ran before userspace and thus doesn't 435 * strictly need retpolines, except for modules which are 436 * loaded late, they very much do need retpoline in their 437 * .init.text 438 */ 439 if (!strcmp(sec->name, ".init.text") && !opts.module) 440 sec->init = true; 441 442 for (offset = 0; offset < sec_size(sec); offset += insn->len) { 443 if (!insns || idx == INSN_CHUNK_MAX) { 444 insns = calloc(INSN_CHUNK_SIZE, sizeof(*insn)); 445 if (!insns) { 446 ERROR_GLIBC("calloc"); 447 return -1; 448 } 449 idx = 0; 450 } else { 451 idx++; 452 } 453 insn = &insns[idx]; 454 insn->idx = idx; 455 456 INIT_LIST_HEAD(&insn->call_node); 457 insn->sec = sec; 458 insn->offset = offset; 459 insn->prev_len = prev_len; 460 461 if (arch_decode_instruction(file, sec, offset, sec_size(sec) - offset, insn)) 462 return -1; 463 464 prev_len = insn->len; 465 466 /* 467 * By default, "ud2" is a dead end unless otherwise 468 * annotated, because GCC 7 inserts it for certain 469 * divide-by-zero cases. 470 */ 471 if (insn->type == INSN_BUG) 472 insn->dead_end = true; 473 474 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset)); 475 nr_insns++; 476 } 477 478 sec_for_each_sym(sec, func) { 479 if (!is_notype_sym(func) && !is_func_sym(func)) 480 continue; 481 482 if (func->offset == sec_size(sec)) { 483 /* Heuristic: likely an "end" symbol */ 484 if (is_notype_sym(func)) 485 continue; 486 ERROR("%s(): STT_FUNC at end of section", func->name); 487 return -1; 488 } 489 490 if (func->embedded_insn || is_alias_sym(func)) 491 continue; 492 493 if (!find_insn(file, sec, func->offset)) { 494 ERROR("%s(): can't find starting instruction", func->name); 495 return -1; 496 } 497 498 sym_for_each_insn(file, func, insn) { 499 insn->sym = func; 500 if (is_func_sym(func) && 501 insn->type == INSN_ENDBR && 502 list_empty(&insn->call_node)) { 503 if (insn->offset == func->offset) { 504 list_add_tail(&insn->call_node, &file->endbr_list); 505 file->nr_endbr++; 506 } else { 507 file->nr_endbr_int++; 508 } 509 } 510 } 511 } 512 } 513 514 if (opts.stats) 515 printf("nr_insns: %lu\n", nr_insns); 516 517 return 0; 518 } 519 520 /* 521 * Known pv_ops*[] arrays. 522 */ 523 static struct { 524 const char *name; 525 int idx_off; 526 } pv_ops_tables[] = { 527 { .name = "pv_ops", }, 528 { .name = "pv_ops_lock", }, 529 { .name = NULL, .idx_off = -1 } 530 }; 531 532 /* 533 * Get index offset for a pv_ops* array. 534 */ 535 int pv_ops_idx_off(const char *symname) 536 { 537 int idx; 538 539 for (idx = 0; pv_ops_tables[idx].name; idx++) { 540 if (!strcmp(symname, pv_ops_tables[idx].name)) 541 break; 542 } 543 544 return pv_ops_tables[idx].idx_off; 545 } 546 547 /* 548 * Read a pv_ops*[] .data table to find the static initialized values. 549 */ 550 static int add_pv_ops(struct objtool_file *file, int pv_ops_idx) 551 { 552 struct symbol *sym, *func; 553 unsigned long off, end; 554 struct reloc *reloc; 555 int idx, idx_off; 556 const char *symname; 557 558 symname = pv_ops_tables[pv_ops_idx].name; 559 sym = find_symbol_by_name(file->elf, symname); 560 if (!sym) { 561 ERROR("Unknown pv_ops array %s", symname); 562 return -1; 563 } 564 565 off = sym->offset; 566 end = off + sym->len; 567 idx_off = pv_ops_tables[pv_ops_idx].idx_off; 568 if (idx_off < 0) { 569 ERROR("pv_ops array %s has unknown index offset", symname); 570 return -1; 571 } 572 573 for (;;) { 574 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 575 if (!reloc) 576 break; 577 578 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long); 579 580 func = reloc->sym; 581 if (is_sec_sym(func)) 582 func = find_symbol_by_offset(reloc->sym->sec, 583 reloc_addend(reloc)); 584 if (!func) { 585 ERROR_FUNC(reloc->sym->sec, reloc_addend(reloc), 586 "can't find func at %s[%d]", symname, idx); 587 return -1; 588 } 589 590 if (objtool_pv_add(file, idx + idx_off, func)) 591 return -1; 592 593 off = reloc_offset(reloc) + 1; 594 if (off > end) 595 break; 596 } 597 598 return 0; 599 } 600 601 /* 602 * Allocate and initialize file->pv_ops[]. 603 */ 604 static int init_pv_ops(struct objtool_file *file) 605 { 606 struct symbol *sym; 607 int idx, nr; 608 609 if (!opts.noinstr) 610 return 0; 611 612 file->pv_ops = NULL; 613 614 nr = 0; 615 for (idx = 0; pv_ops_tables[idx].name; idx++) { 616 sym = find_symbol_by_name(file->elf, pv_ops_tables[idx].name); 617 if (!sym) { 618 pv_ops_tables[idx].idx_off = -1; 619 continue; 620 } 621 pv_ops_tables[idx].idx_off = nr; 622 nr += sym->len / sizeof(unsigned long); 623 } 624 625 if (nr == 0) 626 return 0; 627 628 file->pv_ops = calloc(nr, sizeof(struct pv_state)); 629 if (!file->pv_ops) { 630 ERROR_GLIBC("calloc"); 631 return -1; 632 } 633 634 for (idx = 0; idx < nr; idx++) 635 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 636 637 for (idx = 0; pv_ops_tables[idx].name; idx++) { 638 if (pv_ops_tables[idx].idx_off < 0) 639 continue; 640 if (add_pv_ops(file, idx)) 641 return -1; 642 } 643 644 return 0; 645 } 646 647 static bool is_livepatch_module(struct objtool_file *file) 648 { 649 struct section *sec; 650 651 if (!opts.module) 652 return false; 653 654 sec = find_section_by_name(file->elf, ".modinfo"); 655 if (!sec) 656 return false; 657 658 return memmem(sec->data->d_buf, sec_size(sec), "\0livepatch=Y", 12); 659 } 660 661 static int create_static_call_sections(struct objtool_file *file) 662 { 663 struct static_call_site *site; 664 struct section *sec; 665 struct instruction *insn; 666 struct symbol *key_sym; 667 char *key_name, *tmp; 668 int idx; 669 670 sec = find_section_by_name(file->elf, ".static_call_sites"); 671 if (sec) { 672 /* 673 * Livepatch modules may have already extracted the static call 674 * site entries to take advantage of vmlinux static call 675 * privileges. 676 */ 677 if (!file->klp) 678 WARN("file already has .static_call_sites section, skipping"); 679 680 return 0; 681 } 682 683 if (list_empty(&file->static_call_list)) 684 return 0; 685 686 idx = 0; 687 list_for_each_entry(insn, &file->static_call_list, call_node) 688 idx++; 689 690 sec = elf_create_section_pair(file->elf, ".static_call_sites", 691 sizeof(*site), idx, idx * 2); 692 if (!sec) 693 return -1; 694 695 /* Allow modules to modify the low bits of static_call_site::key */ 696 sec->sh.sh_flags |= SHF_WRITE; 697 698 idx = 0; 699 list_for_each_entry(insn, &file->static_call_list, call_node) { 700 701 /* populate reloc for 'addr' */ 702 if (!elf_init_reloc_text_sym(file->elf, sec, 703 idx * sizeof(*site), idx * 2, 704 insn->sec, insn->offset)) 705 return -1; 706 707 /* find key symbol */ 708 key_name = strdup(insn_call_dest(insn)->name); 709 if (!key_name) { 710 ERROR_GLIBC("strdup"); 711 return -1; 712 } 713 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR, 714 STATIC_CALL_TRAMP_PREFIX_LEN)) { 715 ERROR("static_call: trampoline name malformed: %s", key_name); 716 return -1; 717 } 718 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN; 719 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN); 720 721 key_sym = find_symbol_by_name(file->elf, tmp); 722 if (!key_sym) { 723 if (!opts.module) { 724 ERROR("static_call: can't find static_call_key symbol: %s", tmp); 725 return -1; 726 } 727 728 /* 729 * For modules(), the key might not be exported, which 730 * means the module can make static calls but isn't 731 * allowed to change them. 732 * 733 * In that case we temporarily set the key to be the 734 * trampoline address. This is fixed up in 735 * static_call_add_module(). 736 */ 737 key_sym = insn_call_dest(insn); 738 } 739 740 /* populate reloc for 'key' */ 741 if (!elf_init_reloc_data_sym(file->elf, sec, 742 idx * sizeof(*site) + 4, 743 (idx * 2) + 1, key_sym, 744 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL)) 745 return -1; 746 747 idx++; 748 } 749 750 return 0; 751 } 752 753 static int create_retpoline_sites_sections(struct objtool_file *file) 754 { 755 struct instruction *insn; 756 struct section *sec; 757 int idx; 758 759 sec = find_section_by_name(file->elf, ".retpoline_sites"); 760 if (sec) { 761 WARN("file already has .retpoline_sites, skipping"); 762 return 0; 763 } 764 765 idx = 0; 766 list_for_each_entry(insn, &file->retpoline_call_list, call_node) 767 idx++; 768 769 if (!idx) 770 return 0; 771 772 sec = elf_create_section_pair(file->elf, ".retpoline_sites", 773 sizeof(int), idx, idx); 774 if (!sec) 775 return -1; 776 777 idx = 0; 778 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 779 780 if (!elf_init_reloc_text_sym(file->elf, sec, 781 idx * sizeof(int), idx, 782 insn->sec, insn->offset)) 783 return -1; 784 785 idx++; 786 } 787 788 return 0; 789 } 790 791 static int create_return_sites_sections(struct objtool_file *file) 792 { 793 struct instruction *insn; 794 struct section *sec; 795 int idx; 796 797 sec = find_section_by_name(file->elf, ".return_sites"); 798 if (sec) { 799 WARN("file already has .return_sites, skipping"); 800 return 0; 801 } 802 803 idx = 0; 804 list_for_each_entry(insn, &file->return_thunk_list, call_node) 805 idx++; 806 807 if (!idx) 808 return 0; 809 810 sec = elf_create_section_pair(file->elf, ".return_sites", 811 sizeof(int), idx, idx); 812 if (!sec) 813 return -1; 814 815 idx = 0; 816 list_for_each_entry(insn, &file->return_thunk_list, call_node) { 817 818 if (!elf_init_reloc_text_sym(file->elf, sec, 819 idx * sizeof(int), idx, 820 insn->sec, insn->offset)) 821 return -1; 822 823 idx++; 824 } 825 826 return 0; 827 } 828 829 static int create_ibt_endbr_seal_sections(struct objtool_file *file) 830 { 831 struct instruction *insn; 832 struct section *sec; 833 int idx; 834 835 sec = find_section_by_name(file->elf, ".ibt_endbr_seal"); 836 if (sec) { 837 WARN("file already has .ibt_endbr_seal, skipping"); 838 return 0; 839 } 840 841 idx = 0; 842 list_for_each_entry(insn, &file->endbr_list, call_node) 843 idx++; 844 845 if (opts.stats) { 846 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); 847 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); 848 printf("ibt: superfluous ENDBR: %d\n", idx); 849 } 850 851 if (!idx) 852 return 0; 853 854 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal", 855 sizeof(int), idx, idx); 856 if (!sec) 857 return -1; 858 859 idx = 0; 860 list_for_each_entry(insn, &file->endbr_list, call_node) { 861 862 int *site = (int *)sec->data->d_buf + idx; 863 struct symbol *sym = insn->sym; 864 *site = 0; 865 866 if (opts.module && sym && is_func_sym(sym) && 867 insn->offset == sym->offset && 868 (!strcmp(sym->name, "init_module") || 869 !strcmp(sym->name, "cleanup_module"))) { 870 ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead", 871 sym->name); 872 return -1; 873 } 874 875 if (!elf_init_reloc_text_sym(file->elf, sec, 876 idx * sizeof(int), idx, 877 insn->sec, insn->offset)) 878 return -1; 879 880 idx++; 881 } 882 883 return 0; 884 } 885 886 static int create_cfi_sections(struct objtool_file *file) 887 { 888 struct section *sec; 889 struct symbol *sym; 890 int idx; 891 892 sec = find_section_by_name(file->elf, ".cfi_sites"); 893 if (sec) { 894 WARN("file already has .cfi_sites section, skipping"); 895 return 0; 896 } 897 898 idx = 0; 899 for_each_sym(file->elf, sym) { 900 if (!is_func_sym(sym)) 901 continue; 902 903 if (strncmp(sym->name, "__cfi_", 6)) 904 continue; 905 906 idx++; 907 } 908 909 sec = elf_create_section_pair(file->elf, ".cfi_sites", 910 sizeof(unsigned int), idx, idx); 911 if (!sec) 912 return -1; 913 914 idx = 0; 915 for_each_sym(file->elf, sym) { 916 if (!is_func_sym(sym)) 917 continue; 918 919 if (strncmp(sym->name, "__cfi_", 6)) 920 continue; 921 922 if (!elf_init_reloc_text_sym(file->elf, sec, 923 idx * sizeof(unsigned int), idx, 924 sym->sec, sym->offset)) 925 return -1; 926 927 idx++; 928 } 929 930 return 0; 931 } 932 933 static int create_mcount_loc_sections(struct objtool_file *file) 934 { 935 size_t addr_size = elf_addr_size(file->elf); 936 struct instruction *insn; 937 struct section *sec; 938 int idx; 939 940 sec = find_section_by_name(file->elf, "__mcount_loc"); 941 if (sec) { 942 /* 943 * Livepatch modules have already extracted their __mcount_loc 944 * entries to cover the !CONFIG_FTRACE_MCOUNT_USE_OBJTOOL case. 945 */ 946 if (!file->klp) 947 WARN("file already has __mcount_loc section, skipping"); 948 949 return 0; 950 } 951 952 if (list_empty(&file->mcount_loc_list)) 953 return 0; 954 955 idx = 0; 956 list_for_each_entry(insn, &file->mcount_loc_list, call_node) 957 idx++; 958 959 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size, 960 idx, idx); 961 if (!sec) 962 return -1; 963 964 sec->sh.sh_addralign = addr_size; 965 966 idx = 0; 967 list_for_each_entry(insn, &file->mcount_loc_list, call_node) { 968 969 struct reloc *reloc; 970 971 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx, 972 insn->sec, insn->offset); 973 if (!reloc) 974 return -1; 975 976 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32); 977 978 idx++; 979 } 980 981 return 0; 982 } 983 984 static int create_direct_call_sections(struct objtool_file *file) 985 { 986 struct instruction *insn; 987 struct section *sec; 988 int idx; 989 990 sec = find_section_by_name(file->elf, ".call_sites"); 991 if (sec) { 992 WARN("file already has .call_sites section, skipping"); 993 return 0; 994 } 995 996 if (list_empty(&file->call_list)) 997 return 0; 998 999 idx = 0; 1000 list_for_each_entry(insn, &file->call_list, call_node) 1001 idx++; 1002 1003 sec = elf_create_section_pair(file->elf, ".call_sites", 1004 sizeof(unsigned int), idx, idx); 1005 if (!sec) 1006 return -1; 1007 1008 idx = 0; 1009 list_for_each_entry(insn, &file->call_list, call_node) { 1010 1011 if (!elf_init_reloc_text_sym(file->elf, sec, 1012 idx * sizeof(unsigned int), idx, 1013 insn->sec, insn->offset)) 1014 return -1; 1015 1016 idx++; 1017 } 1018 1019 return 0; 1020 } 1021 1022 /* 1023 * Warnings shouldn't be reported for ignored functions. 1024 */ 1025 static int add_ignores(struct objtool_file *file) 1026 { 1027 struct section *rsec; 1028 struct symbol *func; 1029 struct reloc *reloc; 1030 1031 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard"); 1032 if (!rsec) 1033 return 0; 1034 1035 for_each_reloc(rsec, reloc) { 1036 switch (reloc->sym->type) { 1037 case STT_FUNC: 1038 func = reloc->sym; 1039 break; 1040 1041 case STT_SECTION: 1042 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc)); 1043 if (!func) 1044 continue; 1045 break; 1046 1047 default: 1048 ERROR("unexpected relocation symbol type in %s: %d", 1049 rsec->name, reloc->sym->type); 1050 return -1; 1051 } 1052 1053 func->ignore = true; 1054 if (func->cfunc) 1055 func->cfunc->ignore = true; 1056 } 1057 1058 return 0; 1059 } 1060 1061 /* 1062 * This is a whitelist of functions that is allowed to be called with AC set. 1063 * The list is meant to be minimal and only contains compiler instrumentation 1064 * ABI and a few functions used to implement *_{to,from}_user() functions. 1065 * 1066 * These functions must not directly change AC, but may PUSHF/POPF. 1067 */ 1068 static const char *uaccess_safe_builtin[] = { 1069 /* KASAN */ 1070 "kasan_report", 1071 "kasan_check_range", 1072 /* KASAN out-of-line */ 1073 "__asan_loadN_noabort", 1074 "__asan_load1_noabort", 1075 "__asan_load2_noabort", 1076 "__asan_load4_noabort", 1077 "__asan_load8_noabort", 1078 "__asan_load16_noabort", 1079 "__asan_storeN_noabort", 1080 "__asan_store1_noabort", 1081 "__asan_store2_noabort", 1082 "__asan_store4_noabort", 1083 "__asan_store8_noabort", 1084 "__asan_store16_noabort", 1085 "__kasan_check_read", 1086 "__kasan_check_write", 1087 /* KASAN in-line */ 1088 "__asan_report_load_n_noabort", 1089 "__asan_report_load1_noabort", 1090 "__asan_report_load2_noabort", 1091 "__asan_report_load4_noabort", 1092 "__asan_report_load8_noabort", 1093 "__asan_report_load16_noabort", 1094 "__asan_report_store_n_noabort", 1095 "__asan_report_store1_noabort", 1096 "__asan_report_store2_noabort", 1097 "__asan_report_store4_noabort", 1098 "__asan_report_store8_noabort", 1099 "__asan_report_store16_noabort", 1100 /* KCSAN */ 1101 "__kcsan_check_access", 1102 "__kcsan_mb", 1103 "__kcsan_wmb", 1104 "__kcsan_rmb", 1105 "__kcsan_release", 1106 "kcsan_found_watchpoint", 1107 "kcsan_setup_watchpoint", 1108 "kcsan_check_scoped_accesses", 1109 "kcsan_disable_current", 1110 "kcsan_enable_current_nowarn", 1111 /* KCSAN/TSAN */ 1112 "__tsan_func_entry", 1113 "__tsan_func_exit", 1114 "__tsan_read_range", 1115 "__tsan_write_range", 1116 "__tsan_read1", 1117 "__tsan_read2", 1118 "__tsan_read4", 1119 "__tsan_read8", 1120 "__tsan_read16", 1121 "__tsan_write1", 1122 "__tsan_write2", 1123 "__tsan_write4", 1124 "__tsan_write8", 1125 "__tsan_write16", 1126 "__tsan_read_write1", 1127 "__tsan_read_write2", 1128 "__tsan_read_write4", 1129 "__tsan_read_write8", 1130 "__tsan_read_write16", 1131 "__tsan_volatile_read1", 1132 "__tsan_volatile_read2", 1133 "__tsan_volatile_read4", 1134 "__tsan_volatile_read8", 1135 "__tsan_volatile_read16", 1136 "__tsan_volatile_write1", 1137 "__tsan_volatile_write2", 1138 "__tsan_volatile_write4", 1139 "__tsan_volatile_write8", 1140 "__tsan_volatile_write16", 1141 "__tsan_atomic8_load", 1142 "__tsan_atomic16_load", 1143 "__tsan_atomic32_load", 1144 "__tsan_atomic64_load", 1145 "__tsan_atomic8_store", 1146 "__tsan_atomic16_store", 1147 "__tsan_atomic32_store", 1148 "__tsan_atomic64_store", 1149 "__tsan_atomic8_exchange", 1150 "__tsan_atomic16_exchange", 1151 "__tsan_atomic32_exchange", 1152 "__tsan_atomic64_exchange", 1153 "__tsan_atomic8_fetch_add", 1154 "__tsan_atomic16_fetch_add", 1155 "__tsan_atomic32_fetch_add", 1156 "__tsan_atomic64_fetch_add", 1157 "__tsan_atomic8_fetch_sub", 1158 "__tsan_atomic16_fetch_sub", 1159 "__tsan_atomic32_fetch_sub", 1160 "__tsan_atomic64_fetch_sub", 1161 "__tsan_atomic8_fetch_and", 1162 "__tsan_atomic16_fetch_and", 1163 "__tsan_atomic32_fetch_and", 1164 "__tsan_atomic64_fetch_and", 1165 "__tsan_atomic8_fetch_or", 1166 "__tsan_atomic16_fetch_or", 1167 "__tsan_atomic32_fetch_or", 1168 "__tsan_atomic64_fetch_or", 1169 "__tsan_atomic8_fetch_xor", 1170 "__tsan_atomic16_fetch_xor", 1171 "__tsan_atomic32_fetch_xor", 1172 "__tsan_atomic64_fetch_xor", 1173 "__tsan_atomic8_fetch_nand", 1174 "__tsan_atomic16_fetch_nand", 1175 "__tsan_atomic32_fetch_nand", 1176 "__tsan_atomic64_fetch_nand", 1177 "__tsan_atomic8_compare_exchange_strong", 1178 "__tsan_atomic16_compare_exchange_strong", 1179 "__tsan_atomic32_compare_exchange_strong", 1180 "__tsan_atomic64_compare_exchange_strong", 1181 "__tsan_atomic8_compare_exchange_weak", 1182 "__tsan_atomic16_compare_exchange_weak", 1183 "__tsan_atomic32_compare_exchange_weak", 1184 "__tsan_atomic64_compare_exchange_weak", 1185 "__tsan_atomic8_compare_exchange_val", 1186 "__tsan_atomic16_compare_exchange_val", 1187 "__tsan_atomic32_compare_exchange_val", 1188 "__tsan_atomic64_compare_exchange_val", 1189 "__tsan_atomic_thread_fence", 1190 "__tsan_atomic_signal_fence", 1191 "__tsan_unaligned_read16", 1192 "__tsan_unaligned_write16", 1193 /* KCOV */ 1194 "write_comp_data", 1195 "check_kcov_mode", 1196 "__sanitizer_cov_trace_pc", 1197 "__sanitizer_cov_trace_const_cmp1", 1198 "__sanitizer_cov_trace_const_cmp2", 1199 "__sanitizer_cov_trace_const_cmp4", 1200 "__sanitizer_cov_trace_const_cmp8", 1201 "__sanitizer_cov_trace_cmp1", 1202 "__sanitizer_cov_trace_cmp2", 1203 "__sanitizer_cov_trace_cmp4", 1204 "__sanitizer_cov_trace_cmp8", 1205 "__sanitizer_cov_trace_switch", 1206 /* KMSAN */ 1207 "kmsan_copy_to_user", 1208 "kmsan_disable_current", 1209 "kmsan_enable_current", 1210 "kmsan_report", 1211 "kmsan_unpoison_entry_regs", 1212 "kmsan_unpoison_memory", 1213 "__msan_chain_origin", 1214 "__msan_get_context_state", 1215 "__msan_instrument_asm_store", 1216 "__msan_metadata_ptr_for_load_1", 1217 "__msan_metadata_ptr_for_load_2", 1218 "__msan_metadata_ptr_for_load_4", 1219 "__msan_metadata_ptr_for_load_8", 1220 "__msan_metadata_ptr_for_load_n", 1221 "__msan_metadata_ptr_for_store_1", 1222 "__msan_metadata_ptr_for_store_2", 1223 "__msan_metadata_ptr_for_store_4", 1224 "__msan_metadata_ptr_for_store_8", 1225 "__msan_metadata_ptr_for_store_n", 1226 "__msan_poison_alloca", 1227 "__msan_warning", 1228 /* UBSAN */ 1229 "ubsan_type_mismatch_common", 1230 "__ubsan_handle_type_mismatch", 1231 "__ubsan_handle_type_mismatch_v1", 1232 "__ubsan_handle_shift_out_of_bounds", 1233 "__ubsan_handle_load_invalid_value", 1234 /* KSTACK_ERASE */ 1235 "__sanitizer_cov_stack_depth", 1236 /* TRACE_BRANCH_PROFILING */ 1237 "ftrace_likely_update", 1238 /* STACKPROTECTOR */ 1239 "__stack_chk_fail", 1240 /* misc */ 1241 "csum_partial_copy_generic", 1242 "copy_mc_fragile", 1243 "copy_mc_fragile_handle_tail", 1244 "copy_mc_enhanced_fast_string", 1245 "rep_stos_alternative", 1246 "rep_movs_alternative", 1247 "copy_to_nontemporal", 1248 NULL 1249 }; 1250 1251 static void add_uaccess_safe(struct objtool_file *file) 1252 { 1253 struct symbol *func; 1254 const char **name; 1255 1256 if (!opts.uaccess) 1257 return; 1258 1259 for (name = uaccess_safe_builtin; *name; name++) { 1260 func = find_symbol_by_name(file->elf, *name); 1261 if (!func) 1262 continue; 1263 1264 func->uaccess_safe = true; 1265 } 1266 } 1267 1268 /* 1269 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol 1270 * will be added to the .retpoline_sites section. 1271 */ 1272 __weak bool arch_is_retpoline(struct symbol *sym) 1273 { 1274 return false; 1275 } 1276 1277 /* 1278 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol 1279 * will be added to the .return_sites section. 1280 */ 1281 __weak bool arch_is_rethunk(struct symbol *sym) 1282 { 1283 return false; 1284 } 1285 1286 /* 1287 * Symbols that are embedded inside other instructions, because sometimes crazy 1288 * code exists. These are mostly ignored for validation purposes. 1289 */ 1290 __weak bool arch_is_embedded_insn(struct symbol *sym) 1291 { 1292 return false; 1293 } 1294 1295 struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) 1296 { 1297 struct reloc *reloc; 1298 1299 if (!file || insn->no_reloc || insn->fake) 1300 return NULL; 1301 1302 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 1303 insn->offset, insn->len); 1304 if (!reloc) { 1305 insn->no_reloc = 1; 1306 return NULL; 1307 } 1308 1309 return reloc; 1310 } 1311 1312 static void remove_insn_ops(struct instruction *insn) 1313 { 1314 struct stack_op *op, *next; 1315 1316 for (op = insn->stack_ops; op; op = next) { 1317 next = op->next; 1318 free(op); 1319 } 1320 insn->stack_ops = NULL; 1321 } 1322 1323 static int annotate_call_site(struct objtool_file *file, 1324 struct instruction *insn, bool sibling) 1325 { 1326 struct reloc *reloc = insn_reloc(file, insn); 1327 struct symbol *sym = insn_call_dest(insn); 1328 1329 if (!sym) 1330 sym = reloc->sym; 1331 1332 if (sym->static_call_tramp) { 1333 list_add_tail(&insn->call_node, &file->static_call_list); 1334 return 0; 1335 } 1336 1337 if (sym->retpoline_thunk) { 1338 list_add_tail(&insn->call_node, &file->retpoline_call_list); 1339 return 0; 1340 } 1341 1342 /* 1343 * Many compilers cannot disable KCOV or sanitizer calls with a function 1344 * attribute so they need a little help, NOP out any such calls from 1345 * noinstr text. 1346 */ 1347 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) { 1348 if (reloc) 1349 set_reloc_type(file->elf, reloc, R_NONE); 1350 1351 if (elf_write_insn(file->elf, insn->sec, 1352 insn->offset, insn->len, 1353 sibling ? arch_ret_insn(insn->len) 1354 : arch_nop_insn(insn->len))) { 1355 return -1; 1356 } 1357 1358 insn->type = sibling ? INSN_RETURN : INSN_NOP; 1359 1360 if (sibling) { 1361 /* 1362 * We've replaced the tail-call JMP insn by two new 1363 * insn: RET; INT3, except we only have a single struct 1364 * insn here. Mark it retpoline_safe to avoid the SLS 1365 * warning, instead of adding another insn. 1366 */ 1367 insn->retpoline_safe = true; 1368 } 1369 1370 return 0; 1371 } 1372 1373 if (opts.mcount && sym->fentry) { 1374 if (sibling) 1375 WARN_INSN(insn, "tail call to __fentry__ !?!?"); 1376 if (opts.mnop) { 1377 if (reloc) 1378 set_reloc_type(file->elf, reloc, R_NONE); 1379 1380 if (elf_write_insn(file->elf, insn->sec, 1381 insn->offset, insn->len, 1382 arch_nop_insn(insn->len))) { 1383 return -1; 1384 } 1385 1386 insn->type = INSN_NOP; 1387 } 1388 1389 list_add_tail(&insn->call_node, &file->mcount_loc_list); 1390 return 0; 1391 } 1392 1393 if (insn->type == INSN_CALL && !insn->sec->init && 1394 !insn->_call_dest->embedded_insn) 1395 list_add_tail(&insn->call_node, &file->call_list); 1396 1397 if (!sibling && dead_end_function(file, sym)) 1398 insn->dead_end = true; 1399 1400 return 0; 1401 } 1402 1403 static int add_call_dest(struct objtool_file *file, struct instruction *insn, 1404 struct symbol *dest, bool sibling) 1405 { 1406 insn->_call_dest = dest; 1407 if (!dest) 1408 return 0; 1409 1410 /* 1411 * Whatever stack impact regular CALLs have, should be undone 1412 * by the RETURN of the called function. 1413 * 1414 * Annotated intra-function calls retain the stack_ops but 1415 * are converted to JUMP, see read_intra_function_calls(). 1416 */ 1417 remove_insn_ops(insn); 1418 1419 return annotate_call_site(file, insn, sibling); 1420 } 1421 1422 static int add_retpoline_call(struct objtool_file *file, struct instruction *insn) 1423 { 1424 /* 1425 * Retpoline calls/jumps are really dynamic calls/jumps in disguise, 1426 * so convert them accordingly. 1427 */ 1428 switch (insn->type) { 1429 case INSN_CALL: 1430 insn->type = INSN_CALL_DYNAMIC; 1431 break; 1432 case INSN_JUMP_UNCONDITIONAL: 1433 insn->type = INSN_JUMP_DYNAMIC; 1434 break; 1435 case INSN_JUMP_CONDITIONAL: 1436 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL; 1437 break; 1438 default: 1439 return 0; 1440 } 1441 1442 insn->retpoline_safe = true; 1443 1444 /* 1445 * Whatever stack impact regular CALLs have, should be undone 1446 * by the RETURN of the called function. 1447 * 1448 * Annotated intra-function calls retain the stack_ops but 1449 * are converted to JUMP, see read_intra_function_calls(). 1450 */ 1451 remove_insn_ops(insn); 1452 1453 return annotate_call_site(file, insn, false); 1454 } 1455 1456 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) 1457 { 1458 /* 1459 * Return thunk tail calls are really just returns in disguise, 1460 * so convert them accordingly. 1461 */ 1462 insn->type = INSN_RETURN; 1463 insn->retpoline_safe = true; 1464 1465 if (add) 1466 list_add_tail(&insn->call_node, &file->return_thunk_list); 1467 } 1468 1469 static bool is_first_func_insn(struct objtool_file *file, 1470 struct instruction *insn) 1471 { 1472 struct symbol *func = insn_func(insn); 1473 1474 if (!func) 1475 return false; 1476 1477 if (insn->offset == func->offset) 1478 return true; 1479 1480 /* Allow direct CALL/JMP past ENDBR */ 1481 if (opts.ibt) { 1482 struct instruction *prev = prev_insn_same_sym(file, insn); 1483 1484 if (prev && prev->type == INSN_ENDBR && 1485 insn->offset == func->offset + prev->len) 1486 return true; 1487 } 1488 1489 return false; 1490 } 1491 1492 /* 1493 * Find the destination instructions for all jumps. 1494 */ 1495 static int add_jump_destinations(struct objtool_file *file) 1496 { 1497 struct instruction *insn; 1498 struct reloc *reloc; 1499 1500 for_each_insn(file, insn) { 1501 struct symbol *func = insn_func(insn); 1502 struct instruction *dest_insn; 1503 struct section *dest_sec; 1504 struct symbol *dest_sym; 1505 unsigned long dest_off; 1506 1507 if (!is_static_jump(insn)) 1508 continue; 1509 1510 if (insn->jump_dest) { 1511 /* 1512 * handle_group_alt() may have previously set 1513 * 'jump_dest' for some alternatives. 1514 */ 1515 continue; 1516 } 1517 1518 reloc = insn_reloc(file, insn); 1519 if (!reloc) { 1520 dest_sec = insn->sec; 1521 dest_off = arch_jump_destination(insn); 1522 dest_sym = dest_sec->sym; 1523 } else { 1524 dest_sym = reloc->sym; 1525 if (is_undef_sym(dest_sym)) { 1526 if (dest_sym->retpoline_thunk) { 1527 if (add_retpoline_call(file, insn)) 1528 return -1; 1529 continue; 1530 } 1531 1532 if (dest_sym->return_thunk) { 1533 add_return_call(file, insn, true); 1534 continue; 1535 } 1536 1537 /* External symbol */ 1538 if (func) { 1539 /* External sibling call */ 1540 if (add_call_dest(file, insn, dest_sym, true)) 1541 return -1; 1542 continue; 1543 } 1544 1545 /* Non-func asm code jumping to external symbol */ 1546 continue; 1547 } 1548 1549 dest_sec = dest_sym->sec; 1550 dest_off = dest_sym->offset + arch_insn_adjusted_addend(insn, reloc); 1551 } 1552 1553 dest_insn = find_insn(file, dest_sec, dest_off); 1554 if (!dest_insn) { 1555 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); 1556 1557 /* 1558 * retbleed_untrain_ret() jumps to 1559 * __x86_return_thunk(), but objtool can't find 1560 * the thunk's starting RET instruction, 1561 * because the RET is also in the middle of 1562 * another instruction. Objtool only knows 1563 * about the outer instruction. 1564 */ 1565 if (sym && sym->embedded_insn) { 1566 add_return_call(file, insn, false); 1567 continue; 1568 } 1569 1570 /* 1571 * GCOV/KCOV dead code can jump to the end of 1572 * the function/section. 1573 */ 1574 if (file->ignore_unreachables && func && 1575 dest_sec == insn->sec && 1576 dest_off == func->offset + func->len) 1577 continue; 1578 1579 ERROR_INSN(insn, "can't find jump dest instruction at %s", 1580 offstr(dest_sec, dest_off)); 1581 return -1; 1582 } 1583 1584 if (!dest_sym || is_sec_sym(dest_sym)) { 1585 dest_sym = dest_insn->sym; 1586 if (!dest_sym) 1587 goto set_jump_dest; 1588 } 1589 1590 if (dest_sym->retpoline_thunk && dest_insn->offset == dest_sym->offset) { 1591 if (add_retpoline_call(file, insn)) 1592 return -1; 1593 continue; 1594 } 1595 1596 if (dest_sym->return_thunk && dest_insn->offset == dest_sym->offset) { 1597 add_return_call(file, insn, true); 1598 continue; 1599 } 1600 1601 if (!insn->sym || insn->sym->pfunc == dest_sym->pfunc) 1602 goto set_jump_dest; 1603 1604 /* 1605 * Internal cross-function jump. 1606 */ 1607 1608 if (is_first_func_insn(file, dest_insn)) { 1609 /* Internal sibling call */ 1610 if (add_call_dest(file, insn, dest_sym, true)) 1611 return -1; 1612 continue; 1613 } 1614 1615 set_jump_dest: 1616 insn->jump_dest = dest_insn; 1617 } 1618 1619 return 0; 1620 } 1621 1622 static struct symbol *find_call_destination(struct section *sec, unsigned long offset) 1623 { 1624 struct symbol *call_dest; 1625 1626 call_dest = find_func_by_offset(sec, offset); 1627 if (!call_dest) 1628 call_dest = find_symbol_by_offset(sec, offset); 1629 1630 return call_dest; 1631 } 1632 1633 /* 1634 * Find the destination instructions for all calls. 1635 */ 1636 static int add_call_destinations(struct objtool_file *file) 1637 { 1638 struct instruction *insn; 1639 unsigned long dest_off; 1640 struct symbol *dest; 1641 struct reloc *reloc; 1642 1643 for_each_insn(file, insn) { 1644 struct symbol *func = insn_func(insn); 1645 if (insn->type != INSN_CALL) 1646 continue; 1647 1648 reloc = insn_reloc(file, insn); 1649 if (!reloc) { 1650 dest_off = arch_jump_destination(insn); 1651 dest = find_call_destination(insn->sec, dest_off); 1652 1653 if (add_call_dest(file, insn, dest, false)) 1654 return -1; 1655 1656 if (func && func->ignore) 1657 continue; 1658 1659 if (!insn_call_dest(insn)) { 1660 ERROR_INSN(insn, "unannotated intra-function call"); 1661 return -1; 1662 } 1663 1664 if (func && !is_func_sym(insn_call_dest(insn))) { 1665 ERROR_INSN(insn, "unsupported call to non-function"); 1666 return -1; 1667 } 1668 1669 } else if (is_sec_sym(reloc->sym)) { 1670 dest_off = arch_insn_adjusted_addend(insn, reloc); 1671 dest = find_call_destination(reloc->sym->sec, dest_off); 1672 if (!dest) { 1673 ERROR_INSN(insn, "can't find call dest symbol at %s+0x%lx", 1674 reloc->sym->sec->name, dest_off); 1675 return -1; 1676 } 1677 1678 if (add_call_dest(file, insn, dest, false)) 1679 return -1; 1680 1681 } else if (reloc->sym->retpoline_thunk) { 1682 if (add_retpoline_call(file, insn)) 1683 return -1; 1684 1685 } else { 1686 if (add_call_dest(file, insn, reloc->sym, false)) 1687 return -1; 1688 } 1689 } 1690 1691 return 0; 1692 } 1693 1694 /* 1695 * The .alternatives section requires some extra special care over and above 1696 * other special sections because alternatives are patched in place. 1697 */ 1698 static int handle_group_alt(struct objtool_file *file, 1699 struct special_alt *special_alt, 1700 struct instruction *orig_insn, 1701 struct instruction **new_insn) 1702 { 1703 struct instruction *last_new_insn = NULL, *insn, *nop = NULL; 1704 struct alt_group *orig_alt_group, *new_alt_group; 1705 unsigned long dest_off; 1706 1707 orig_alt_group = orig_insn->alt_group; 1708 if (!orig_alt_group) { 1709 struct instruction *last_orig_insn = NULL; 1710 1711 orig_alt_group = calloc(1, sizeof(*orig_alt_group)); 1712 if (!orig_alt_group) { 1713 ERROR_GLIBC("calloc"); 1714 return -1; 1715 } 1716 orig_alt_group->cfi = calloc(special_alt->orig_len, 1717 sizeof(struct cfi_state *)); 1718 if (!orig_alt_group->cfi) { 1719 ERROR_GLIBC("calloc"); 1720 return -1; 1721 } 1722 1723 insn = orig_insn; 1724 sec_for_each_insn_from(file, insn) { 1725 if (insn->offset >= special_alt->orig_off + special_alt->orig_len) 1726 break; 1727 1728 insn->alt_group = orig_alt_group; 1729 last_orig_insn = insn; 1730 } 1731 orig_alt_group->orig_group = NULL; 1732 orig_alt_group->first_insn = orig_insn; 1733 orig_alt_group->last_insn = last_orig_insn; 1734 orig_alt_group->nop = NULL; 1735 orig_alt_group->ignore = orig_insn->ignore_alts; 1736 orig_alt_group->feature = 0; 1737 } else { 1738 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - 1739 orig_alt_group->first_insn->offset != special_alt->orig_len) { 1740 ERROR_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d", 1741 orig_alt_group->last_insn->offset + 1742 orig_alt_group->last_insn->len - 1743 orig_alt_group->first_insn->offset, 1744 special_alt->orig_len); 1745 return -1; 1746 } 1747 } 1748 1749 new_alt_group = calloc(1, sizeof(*new_alt_group)); 1750 if (!new_alt_group) { 1751 ERROR_GLIBC("calloc"); 1752 return -1; 1753 } 1754 1755 if (special_alt->new_len < special_alt->orig_len) { 1756 /* 1757 * Insert a fake nop at the end to make the replacement 1758 * alt_group the same size as the original. This is needed to 1759 * allow propagate_alt_cfi() to do its magic. When the last 1760 * instruction affects the stack, the instruction after it (the 1761 * nop) will propagate the new state to the shared CFI array. 1762 */ 1763 nop = calloc(1, sizeof(*nop)); 1764 if (!nop) { 1765 ERROR_GLIBC("calloc"); 1766 return -1; 1767 } 1768 memset(nop, 0, sizeof(*nop)); 1769 1770 nop->sec = special_alt->new_sec; 1771 nop->offset = special_alt->new_off + special_alt->new_len; 1772 nop->len = special_alt->orig_len - special_alt->new_len; 1773 nop->type = INSN_NOP; 1774 nop->sym = orig_insn->sym; 1775 nop->alt_group = new_alt_group; 1776 nop->fake = 1; 1777 } 1778 1779 if (!special_alt->new_len) { 1780 *new_insn = nop; 1781 goto end; 1782 } 1783 1784 insn = *new_insn; 1785 sec_for_each_insn_from(file, insn) { 1786 struct reloc *alt_reloc; 1787 1788 if (insn->offset >= special_alt->new_off + special_alt->new_len) 1789 break; 1790 1791 last_new_insn = insn; 1792 1793 insn->sym = orig_insn->sym; 1794 insn->alt_group = new_alt_group; 1795 1796 /* 1797 * Since alternative replacement code is copy/pasted by the 1798 * kernel after applying relocations, generally such code can't 1799 * have relative-address relocation references to outside the 1800 * .altinstr_replacement section, unless the arch's 1801 * alternatives code can adjust the relative offsets 1802 * accordingly. 1803 */ 1804 alt_reloc = insn_reloc(file, insn); 1805 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) && 1806 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { 1807 1808 ERROR_INSN(insn, "unsupported relocation in alternatives section"); 1809 return -1; 1810 } 1811 1812 if (!is_static_jump(insn)) 1813 continue; 1814 1815 if (!insn->immediate) 1816 continue; 1817 1818 dest_off = arch_jump_destination(insn); 1819 if (dest_off == special_alt->new_off + special_alt->new_len) { 1820 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn); 1821 if (!insn->jump_dest) { 1822 ERROR_INSN(insn, "can't find alternative jump destination"); 1823 return -1; 1824 } 1825 } 1826 } 1827 1828 if (!last_new_insn) { 1829 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1830 "can't find last new alternative instruction"); 1831 return -1; 1832 } 1833 1834 end: 1835 new_alt_group->orig_group = orig_alt_group; 1836 new_alt_group->first_insn = *new_insn; 1837 new_alt_group->last_insn = last_new_insn; 1838 new_alt_group->nop = nop; 1839 new_alt_group->ignore = (*new_insn)->ignore_alts; 1840 new_alt_group->cfi = orig_alt_group->cfi; 1841 new_alt_group->feature = special_alt->feature; 1842 return 0; 1843 } 1844 1845 /* 1846 * A jump table entry can either convert a nop to a jump or a jump to a nop. 1847 * If the original instruction is a jump, make the alt entry an effective nop 1848 * by just skipping the original instruction. 1849 */ 1850 static int handle_jump_alt(struct objtool_file *file, 1851 struct special_alt *special_alt, 1852 struct instruction *orig_insn, 1853 struct instruction **new_insn) 1854 { 1855 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && 1856 orig_insn->type != INSN_NOP) { 1857 1858 ERROR_INSN(orig_insn, "unsupported instruction at jump label"); 1859 return -1; 1860 } 1861 1862 if (opts.hack_jump_label && special_alt->key_addend & 2) { 1863 struct reloc *reloc = insn_reloc(file, orig_insn); 1864 1865 if (reloc) 1866 set_reloc_type(file->elf, reloc, R_NONE); 1867 1868 if (elf_write_insn(file->elf, orig_insn->sec, 1869 orig_insn->offset, orig_insn->len, 1870 arch_nop_insn(orig_insn->len))) { 1871 return -1; 1872 } 1873 1874 orig_insn->type = INSN_NOP; 1875 } 1876 1877 if (orig_insn->type == INSN_NOP) { 1878 if (orig_insn->len == 2) 1879 file->jl_nop_short++; 1880 else 1881 file->jl_nop_long++; 1882 1883 return 0; 1884 } 1885 1886 if (orig_insn->len == 2) 1887 file->jl_short++; 1888 else 1889 file->jl_long++; 1890 1891 *new_insn = next_insn_same_sec(file, orig_insn); 1892 return 0; 1893 } 1894 1895 /* 1896 * Read all the special sections which have alternate instructions which can be 1897 * patched in or redirected to at runtime. Each instruction having alternate 1898 * instruction(s) has them added to its insn->alts list, which will be 1899 * traversed in validate_branch(). 1900 */ 1901 static int add_special_section_alts(struct objtool_file *file) 1902 { 1903 struct list_head special_alts; 1904 struct instruction *orig_insn, *new_insn; 1905 struct special_alt *special_alt, *tmp; 1906 enum alternative_type alt_type; 1907 struct alternative *alt; 1908 struct alternative *a; 1909 1910 if (special_get_alts(file->elf, &special_alts)) 1911 return -1; 1912 1913 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) { 1914 1915 orig_insn = find_insn(file, special_alt->orig_sec, 1916 special_alt->orig_off); 1917 if (!orig_insn) { 1918 ERROR_FUNC(special_alt->orig_sec, special_alt->orig_off, 1919 "special: can't find orig instruction"); 1920 return -1; 1921 } 1922 1923 new_insn = NULL; 1924 if (!special_alt->group || special_alt->new_len) { 1925 new_insn = find_insn(file, special_alt->new_sec, 1926 special_alt->new_off); 1927 if (!new_insn) { 1928 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1929 "special: can't find new instruction"); 1930 return -1; 1931 } 1932 } 1933 1934 if (special_alt->group) { 1935 if (!special_alt->orig_len) { 1936 ERROR_INSN(orig_insn, "empty alternative entry"); 1937 continue; 1938 } 1939 1940 if (handle_group_alt(file, special_alt, orig_insn, &new_insn)) 1941 return -1; 1942 1943 alt_type = ALT_TYPE_INSTRUCTIONS; 1944 1945 } else if (special_alt->jump_or_nop) { 1946 if (handle_jump_alt(file, special_alt, orig_insn, &new_insn)) 1947 return -1; 1948 1949 alt_type = ALT_TYPE_JUMP_TABLE; 1950 } else { 1951 alt_type = ALT_TYPE_EX_TABLE; 1952 } 1953 1954 alt = calloc(1, sizeof(*alt)); 1955 if (!alt) { 1956 ERROR_GLIBC("calloc"); 1957 return -1; 1958 } 1959 1960 alt->insn = new_insn; 1961 alt->type = alt_type; 1962 alt->next = NULL; 1963 1964 /* 1965 * Store alternatives in the same order they have been 1966 * defined. 1967 */ 1968 if (!orig_insn->alts) { 1969 orig_insn->alts = alt; 1970 } else { 1971 for (a = orig_insn->alts; a->next; a = a->next) 1972 ; 1973 a->next = alt; 1974 } 1975 1976 list_del(&special_alt->list); 1977 free(special_alt); 1978 } 1979 1980 if (opts.stats) { 1981 printf("jl\\\tNOP\tJMP\n"); 1982 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); 1983 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); 1984 } 1985 1986 return 0; 1987 } 1988 1989 __weak unsigned long arch_jump_table_sym_offset(struct reloc *reloc, struct reloc *table) 1990 { 1991 return reloc->sym->offset + reloc_addend(reloc); 1992 } 1993 1994 static int add_jump_table(struct objtool_file *file, struct instruction *insn) 1995 { 1996 unsigned long table_size = insn_jump_table_size(insn); 1997 struct symbol *pfunc = insn_func(insn)->pfunc; 1998 struct reloc *table = insn_jump_table(insn); 1999 struct instruction *dest_insn; 2000 unsigned int prev_offset = 0; 2001 struct reloc *reloc = table; 2002 struct alternative *alt; 2003 unsigned long sym_offset; 2004 2005 /* 2006 * Each @reloc is a switch table relocation which points to the target 2007 * instruction. 2008 */ 2009 for_each_reloc_from(table->sec, reloc) { 2010 2011 /* Check for the end of the table: */ 2012 if (table_size && reloc_offset(reloc) - reloc_offset(table) >= table_size) 2013 break; 2014 if (reloc != table && is_jump_table(reloc)) 2015 break; 2016 2017 /* Make sure the table entries are consecutive: */ 2018 if (prev_offset && reloc_offset(reloc) != prev_offset + arch_reloc_size(reloc)) 2019 break; 2020 2021 sym_offset = arch_jump_table_sym_offset(reloc, table); 2022 2023 /* Detect function pointers from contiguous objects: */ 2024 if (reloc->sym->sec == pfunc->sec && sym_offset == pfunc->offset) 2025 break; 2026 2027 /* 2028 * Clang sometimes leaves dangling unused jump table entries 2029 * which point to the end of the function. Ignore them. 2030 */ 2031 if (reloc->sym->sec == pfunc->sec && 2032 sym_offset == pfunc->offset + pfunc->len) 2033 goto next; 2034 2035 dest_insn = find_insn(file, reloc->sym->sec, sym_offset); 2036 if (!dest_insn) 2037 break; 2038 2039 /* Make sure the destination is in the same function: */ 2040 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc) 2041 break; 2042 2043 alt = calloc(1, sizeof(*alt)); 2044 if (!alt) { 2045 ERROR_GLIBC("calloc"); 2046 return -1; 2047 } 2048 2049 alt->insn = dest_insn; 2050 alt->next = insn->alts; 2051 insn->alts = alt; 2052 next: 2053 prev_offset = reloc_offset(reloc); 2054 } 2055 2056 if (!prev_offset) { 2057 ERROR_INSN(insn, "can't find switch jump table"); 2058 return -1; 2059 } 2060 2061 return 0; 2062 } 2063 2064 /* 2065 * find_jump_table() - Given a dynamic jump, find the switch jump table 2066 * associated with it. 2067 */ 2068 static void find_jump_table(struct objtool_file *file, struct symbol *func, 2069 struct instruction *insn) 2070 { 2071 struct reloc *table_reloc; 2072 struct instruction *dest_insn, *orig_insn = insn; 2073 unsigned long table_size; 2074 unsigned long sym_offset; 2075 2076 /* 2077 * Backward search using the @first_jump_src links, these help avoid 2078 * much of the 'in between' code. Which avoids us getting confused by 2079 * it. 2080 */ 2081 for (; 2082 insn && insn_func(insn) && insn_func(insn)->pfunc == func; 2083 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) { 2084 2085 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC) 2086 break; 2087 2088 /* allow small jumps within the range */ 2089 if (insn->type == INSN_JUMP_UNCONDITIONAL && 2090 insn->jump_dest && 2091 (insn->jump_dest->offset <= insn->offset || 2092 insn->jump_dest->offset > orig_insn->offset)) 2093 break; 2094 2095 table_reloc = arch_find_switch_table(file, insn, &table_size); 2096 if (!table_reloc) 2097 continue; 2098 2099 sym_offset = table_reloc->sym->offset + reloc_addend(table_reloc); 2100 2101 dest_insn = find_insn(file, table_reloc->sym->sec, sym_offset); 2102 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func) 2103 continue; 2104 2105 set_jump_table(table_reloc); 2106 orig_insn->_jump_table = table_reloc; 2107 orig_insn->_jump_table_size = table_size; 2108 2109 break; 2110 } 2111 } 2112 2113 /* 2114 * First pass: Mark the head of each jump table so that in the next pass, 2115 * we know when a given jump table ends and the next one starts. 2116 */ 2117 static void mark_func_jump_tables(struct objtool_file *file, 2118 struct symbol *func) 2119 { 2120 struct instruction *insn, *last = NULL; 2121 2122 func_for_each_insn(file, func, insn) { 2123 if (!last) 2124 last = insn; 2125 2126 /* 2127 * Store back-pointers for forward jumps such 2128 * that find_jump_table() can back-track using those and 2129 * avoid some potentially confusing code. 2130 */ 2131 if (insn->jump_dest && 2132 insn->jump_dest->offset > insn->offset && 2133 !insn->jump_dest->first_jump_src) { 2134 2135 insn->jump_dest->first_jump_src = insn; 2136 last = insn->jump_dest; 2137 } 2138 2139 if (insn->type != INSN_JUMP_DYNAMIC) 2140 continue; 2141 2142 find_jump_table(file, func, insn); 2143 } 2144 } 2145 2146 static int add_func_jump_tables(struct objtool_file *file, 2147 struct symbol *func) 2148 { 2149 struct instruction *insn; 2150 2151 func_for_each_insn(file, func, insn) { 2152 if (!insn_jump_table(insn)) 2153 continue; 2154 2155 if (add_jump_table(file, insn)) 2156 return -1; 2157 } 2158 2159 return 0; 2160 } 2161 2162 /* 2163 * For some switch statements, gcc generates a jump table in the .rodata 2164 * section which contains a list of addresses within the function to jump to. 2165 * This finds these jump tables and adds them to the insn->alts lists. 2166 */ 2167 static int add_jump_table_alts(struct objtool_file *file) 2168 { 2169 struct symbol *func; 2170 2171 if (!file->rodata) 2172 return 0; 2173 2174 for_each_sym(file->elf, func) { 2175 if (!is_func_sym(func) || is_alias_sym(func)) 2176 continue; 2177 2178 mark_func_jump_tables(file, func); 2179 if (add_func_jump_tables(file, func)) 2180 return -1; 2181 } 2182 2183 return 0; 2184 } 2185 2186 static void set_func_state(struct cfi_state *state) 2187 { 2188 state->cfa = initial_func_cfi.cfa; 2189 memcpy(&state->regs, &initial_func_cfi.regs, 2190 CFI_NUM_REGS * sizeof(struct cfi_reg)); 2191 state->stack_size = initial_func_cfi.cfa.offset; 2192 state->type = UNWIND_HINT_TYPE_CALL; 2193 } 2194 2195 static int read_unwind_hints(struct objtool_file *file) 2196 { 2197 struct cfi_state cfi = init_cfi; 2198 struct section *sec; 2199 struct unwind_hint *hint; 2200 struct instruction *insn; 2201 struct reloc *reloc; 2202 unsigned long offset; 2203 int i; 2204 2205 sec = find_section_by_name(file->elf, ".discard.unwind_hints"); 2206 if (!sec) 2207 return 0; 2208 2209 if (!sec->rsec) { 2210 ERROR("missing .rela.discard.unwind_hints section"); 2211 return -1; 2212 } 2213 2214 if (sec_size(sec) % sizeof(struct unwind_hint)) { 2215 ERROR("struct unwind_hint size mismatch"); 2216 return -1; 2217 } 2218 2219 file->hints = true; 2220 2221 for (i = 0; i < sec_size(sec) / sizeof(struct unwind_hint); i++) { 2222 hint = (struct unwind_hint *)sec->data->d_buf + i; 2223 2224 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); 2225 if (!reloc) { 2226 ERROR("can't find reloc for unwind_hints[%d]", i); 2227 return -1; 2228 } 2229 2230 offset = reloc->sym->offset + reloc_addend(reloc); 2231 2232 insn = find_insn(file, reloc->sym->sec, offset); 2233 if (!insn) { 2234 ERROR("can't find insn for unwind_hints[%d]", i); 2235 return -1; 2236 } 2237 2238 insn->hint = true; 2239 2240 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) { 2241 insn->cfi = &force_undefined_cfi; 2242 continue; 2243 } 2244 2245 if (hint->type == UNWIND_HINT_TYPE_SAVE) { 2246 insn->hint = false; 2247 insn->save = true; 2248 continue; 2249 } 2250 2251 if (hint->type == UNWIND_HINT_TYPE_RESTORE) { 2252 insn->restore = true; 2253 continue; 2254 } 2255 2256 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 2257 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 2258 2259 if (sym && is_global_sym(sym)) { 2260 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { 2261 ERROR_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR"); 2262 return -1; 2263 } 2264 } 2265 } 2266 2267 if (hint->type == UNWIND_HINT_TYPE_FUNC) { 2268 insn->cfi = &func_cfi; 2269 continue; 2270 } 2271 2272 if (insn->cfi) 2273 cfi = *(insn->cfi); 2274 2275 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { 2276 ERROR_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg); 2277 return -1; 2278 } 2279 2280 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); 2281 cfi.type = hint->type; 2282 cfi.signal = hint->signal; 2283 2284 insn->cfi = cfi_hash_find_or_add(&cfi); 2285 } 2286 2287 return 0; 2288 } 2289 2290 static int read_annotate(struct objtool_file *file, 2291 int (*func)(struct objtool_file *file, int type, struct instruction *insn)) 2292 { 2293 struct section *sec; 2294 struct instruction *insn; 2295 struct reloc *reloc; 2296 uint64_t offset; 2297 int type; 2298 2299 sec = find_section_by_name(file->elf, ".discard.annotate_insn"); 2300 if (!sec) 2301 return 0; 2302 2303 if (!sec->rsec) 2304 return 0; 2305 2306 if (sec->sh.sh_entsize != 8) { 2307 static bool warned = false; 2308 if (!warned && opts.verbose) { 2309 WARN("%s: dodgy linker, sh_entsize != 8", sec->name); 2310 warned = true; 2311 } 2312 sec->sh.sh_entsize = 8; 2313 } 2314 2315 if (sec_num_entries(sec) != sec_num_entries(sec->rsec)) { 2316 ERROR("bad .discard.annotate_insn section: missing relocs"); 2317 return -1; 2318 } 2319 2320 for_each_reloc(sec->rsec, reloc) { 2321 type = annotype(file->elf, sec, reloc); 2322 offset = reloc->sym->offset + reloc_addend(reloc); 2323 insn = find_insn(file, reloc->sym->sec, offset); 2324 2325 if (!insn) { 2326 ERROR("bad .discard.annotate_insn entry: %d of type %d", reloc_idx(reloc), type); 2327 return -1; 2328 } 2329 2330 if (func(file, type, insn)) 2331 return -1; 2332 } 2333 2334 return 0; 2335 } 2336 2337 static int __annotate_early(struct objtool_file *file, int type, struct instruction *insn) 2338 { 2339 switch (type) { 2340 2341 /* Must be before add_special_section_alts() */ 2342 case ANNOTYPE_IGNORE_ALTS: 2343 insn->ignore_alts = true; 2344 break; 2345 2346 /* 2347 * Must be before read_unwind_hints() since that needs insn->noendbr. 2348 */ 2349 case ANNOTYPE_NOENDBR: 2350 insn->noendbr = 1; 2351 break; 2352 2353 default: 2354 break; 2355 } 2356 2357 return 0; 2358 } 2359 2360 static int __annotate_ifc(struct objtool_file *file, int type, struct instruction *insn) 2361 { 2362 unsigned long dest_off; 2363 2364 if (type != ANNOTYPE_INTRA_FUNCTION_CALL) 2365 return 0; 2366 2367 if (insn->type != INSN_CALL) { 2368 ERROR_INSN(insn, "intra_function_call not a direct call"); 2369 return -1; 2370 } 2371 2372 /* 2373 * Treat intra-function CALLs as JMPs, but with a stack_op. 2374 * See add_call_destinations(), which strips stack_ops from 2375 * normal CALLs. 2376 */ 2377 insn->type = INSN_JUMP_UNCONDITIONAL; 2378 2379 dest_off = arch_jump_destination(insn); 2380 insn->jump_dest = find_insn(file, insn->sec, dest_off); 2381 if (!insn->jump_dest) { 2382 ERROR_INSN(insn, "can't find call dest at %s+0x%lx", 2383 insn->sec->name, dest_off); 2384 return -1; 2385 } 2386 2387 return 0; 2388 } 2389 2390 static int __annotate_late(struct objtool_file *file, int type, struct instruction *insn) 2391 { 2392 struct symbol *sym; 2393 2394 switch (type) { 2395 case ANNOTYPE_NOENDBR: 2396 /* early */ 2397 break; 2398 2399 case ANNOTYPE_RETPOLINE_SAFE: 2400 if (insn->type != INSN_JUMP_DYNAMIC && 2401 insn->type != INSN_CALL_DYNAMIC && 2402 insn->type != INSN_RETURN && 2403 insn->type != INSN_NOP) { 2404 ERROR_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); 2405 return -1; 2406 } 2407 2408 insn->retpoline_safe = true; 2409 break; 2410 2411 case ANNOTYPE_INSTR_BEGIN: 2412 insn->instr++; 2413 break; 2414 2415 case ANNOTYPE_INSTR_END: 2416 insn->instr--; 2417 break; 2418 2419 case ANNOTYPE_UNRET_BEGIN: 2420 insn->unret = 1; 2421 break; 2422 2423 case ANNOTYPE_IGNORE_ALTS: 2424 /* early */ 2425 break; 2426 2427 case ANNOTYPE_INTRA_FUNCTION_CALL: 2428 /* ifc */ 2429 break; 2430 2431 case ANNOTYPE_REACHABLE: 2432 insn->dead_end = false; 2433 break; 2434 2435 case ANNOTYPE_NOCFI: 2436 sym = insn->sym; 2437 if (!sym) { 2438 ERROR_INSN(insn, "dodgy NOCFI annotation"); 2439 return -1; 2440 } 2441 insn->sym->nocfi = 1; 2442 break; 2443 2444 default: 2445 ERROR_INSN(insn, "Unknown annotation type: %d", type); 2446 return -1; 2447 } 2448 2449 return 0; 2450 } 2451 2452 /* 2453 * Return true if name matches an instrumentation function, where calls to that 2454 * function from noinstr code can safely be removed, but compilers won't do so. 2455 */ 2456 static bool is_profiling_func(const char *name) 2457 { 2458 /* 2459 * Many compilers cannot disable KCOV with a function attribute. 2460 */ 2461 if (!strncmp(name, "__sanitizer_cov_", 16)) 2462 return true; 2463 2464 return false; 2465 } 2466 2467 static int classify_symbols(struct objtool_file *file) 2468 { 2469 struct symbol *func; 2470 size_t len; 2471 2472 for_each_sym(file->elf, func) { 2473 if (is_notype_sym(func) && strstarts(func->name, ".L")) 2474 func->local_label = true; 2475 2476 if (!is_global_sym(func)) 2477 continue; 2478 2479 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, 2480 strlen(STATIC_CALL_TRAMP_PREFIX_STR))) 2481 func->static_call_tramp = true; 2482 2483 if (arch_is_retpoline(func)) 2484 func->retpoline_thunk = true; 2485 2486 if (arch_is_rethunk(func)) 2487 func->return_thunk = true; 2488 2489 if (arch_is_embedded_insn(func)) 2490 func->embedded_insn = true; 2491 2492 if (arch_ftrace_match(func->name)) 2493 func->fentry = true; 2494 2495 if (is_profiling_func(func->name)) 2496 func->profiling_func = true; 2497 2498 len = strlen(func->name); 2499 if (len > sym_name_max_len) 2500 sym_name_max_len = len; 2501 } 2502 2503 return 0; 2504 } 2505 2506 static void mark_rodata(struct objtool_file *file) 2507 { 2508 struct section *sec; 2509 2510 /* 2511 * Search for the following rodata sections, each of which can 2512 * potentially contain jump tables: 2513 * 2514 * - .rodata: can contain GCC switch tables 2515 * - .rodata.<func>: same, if -fdata-sections is being used 2516 * - .data.rel.ro.c_jump_table: contains C annotated jump tables 2517 * 2518 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2519 */ 2520 for_each_sec(file->elf, sec) { 2521 if (is_rodata_sec(sec)) { 2522 file->rodata = true; 2523 return; 2524 } 2525 } 2526 } 2527 2528 static void mark_holes(struct objtool_file *file) 2529 { 2530 struct instruction *insn; 2531 bool in_hole = false; 2532 2533 if (!opts.link) 2534 return; 2535 2536 /* 2537 * Whole archive runs might encounter dead code from weak symbols. 2538 * This is where the linker will have dropped the weak symbol in 2539 * favour of a regular symbol, but leaves the code in place. 2540 */ 2541 for_each_insn(file, insn) { 2542 if (insn->sym || !find_symbol_hole_containing(insn->sec, insn->offset)) { 2543 in_hole = false; 2544 continue; 2545 } 2546 2547 /* Skip function padding and pfx code */ 2548 if (!in_hole && insn->type == INSN_NOP) 2549 continue; 2550 2551 in_hole = true; 2552 insn->hole = 1; 2553 2554 /* 2555 * If this hole jumps to a .cold function, mark it ignore. 2556 */ 2557 if (insn->jump_dest) { 2558 struct symbol *dest_func = insn_func(insn->jump_dest); 2559 2560 if (dest_func && is_cold_func(dest_func)) 2561 dest_func->ignore = true; 2562 } 2563 } 2564 } 2565 2566 static bool validate_branch_enabled(void) 2567 { 2568 return opts.stackval || 2569 opts.orc || 2570 opts.uaccess; 2571 } 2572 2573 static bool alts_needed(void) 2574 { 2575 return validate_branch_enabled() || 2576 opts.noinstr || 2577 opts.hack_jump_label || 2578 opts.disas || 2579 opts.checksum; 2580 } 2581 2582 int decode_file(struct objtool_file *file) 2583 { 2584 arch_initial_func_cfi_state(&initial_func_cfi); 2585 init_cfi_state(&init_cfi); 2586 init_cfi_state(&func_cfi); 2587 set_func_state(&func_cfi); 2588 init_cfi_state(&force_undefined_cfi); 2589 force_undefined_cfi.force_undefined = true; 2590 2591 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) 2592 return -1; 2593 2594 cfi_hash_add(&init_cfi); 2595 cfi_hash_add(&func_cfi); 2596 2597 file->klp = is_livepatch_module(file); 2598 2599 mark_rodata(file); 2600 2601 if (init_pv_ops(file)) 2602 return -1; 2603 2604 /* 2605 * Must be before add_{jump_call}_destination. 2606 */ 2607 if (classify_symbols(file)) 2608 return -1; 2609 2610 if (decode_instructions(file)) 2611 return -1; 2612 2613 if (add_ignores(file)) 2614 return -1; 2615 2616 add_uaccess_safe(file); 2617 2618 if (read_annotate(file, __annotate_early)) 2619 return -1; 2620 2621 /* 2622 * Must be before add_jump_destinations(), which depends on 'func' 2623 * being set for alternatives, to enable proper sibling call detection. 2624 */ 2625 if (alts_needed()) { 2626 if (add_special_section_alts(file)) 2627 return -1; 2628 } 2629 2630 if (add_jump_destinations(file)) 2631 return -1; 2632 2633 /* 2634 * Must be before add_call_destination(); it changes INSN_CALL to 2635 * INSN_JUMP. 2636 */ 2637 if (read_annotate(file, __annotate_ifc)) 2638 return -1; 2639 2640 if (add_call_destinations(file)) 2641 return -1; 2642 2643 if (add_jump_table_alts(file)) 2644 return -1; 2645 2646 if (read_unwind_hints(file)) 2647 return -1; 2648 2649 /* Must be after add_jump_destinations() */ 2650 mark_holes(file); 2651 2652 /* 2653 * Must be after add_call_destinations() such that it can override 2654 * dead_end_function() marks. 2655 */ 2656 if (read_annotate(file, __annotate_late)) 2657 return -1; 2658 2659 return 0; 2660 } 2661 2662 static bool is_special_call(struct instruction *insn) 2663 { 2664 if (insn->type == INSN_CALL) { 2665 struct symbol *dest = insn_call_dest(insn); 2666 2667 if (!dest) 2668 return false; 2669 2670 if (dest->fentry || dest->embedded_insn) 2671 return true; 2672 } 2673 2674 return false; 2675 } 2676 2677 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state) 2678 { 2679 struct cfi_state *cfi = &state->cfi; 2680 int i; 2681 2682 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap) 2683 return true; 2684 2685 if (cfi->cfa.offset != initial_func_cfi.cfa.offset) 2686 return true; 2687 2688 if (cfi->stack_size != initial_func_cfi.cfa.offset) 2689 return true; 2690 2691 for (i = 0; i < CFI_NUM_REGS; i++) { 2692 if (cfi->regs[i].base != initial_func_cfi.regs[i].base || 2693 cfi->regs[i].offset != initial_func_cfi.regs[i].offset) 2694 return true; 2695 } 2696 2697 return false; 2698 } 2699 2700 static bool check_reg_frame_pos(const struct cfi_reg *reg, 2701 int expected_offset) 2702 { 2703 return reg->base == CFI_CFA && 2704 reg->offset == expected_offset; 2705 } 2706 2707 static bool has_valid_stack_frame(struct insn_state *state) 2708 { 2709 struct cfi_state *cfi = &state->cfi; 2710 2711 if (cfi->cfa.base == CFI_BP && 2712 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) && 2713 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8)) 2714 return true; 2715 2716 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP) 2717 return true; 2718 2719 return false; 2720 } 2721 2722 static int update_cfi_state_regs(struct instruction *insn, 2723 struct cfi_state *cfi, 2724 struct stack_op *op) 2725 { 2726 struct cfi_reg *cfa = &cfi->cfa; 2727 2728 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT) 2729 return 0; 2730 2731 /* push */ 2732 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF) 2733 cfa->offset += 8; 2734 2735 /* pop */ 2736 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF) 2737 cfa->offset -= 8; 2738 2739 /* add immediate to sp */ 2740 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD && 2741 op->dest.reg == CFI_SP && op->src.reg == CFI_SP) 2742 cfa->offset -= op->src.offset; 2743 2744 return 0; 2745 } 2746 2747 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset) 2748 { 2749 if (arch_callee_saved_reg(reg) && 2750 cfi->regs[reg].base == CFI_UNDEFINED) { 2751 cfi->regs[reg].base = base; 2752 cfi->regs[reg].offset = offset; 2753 } 2754 } 2755 2756 static void restore_reg(struct cfi_state *cfi, unsigned char reg) 2757 { 2758 cfi->regs[reg].base = initial_func_cfi.regs[reg].base; 2759 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset; 2760 } 2761 2762 /* 2763 * A note about DRAP stack alignment: 2764 * 2765 * GCC has the concept of a DRAP register, which is used to help keep track of 2766 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP 2767 * register. The typical DRAP pattern is: 2768 * 2769 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10 2770 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp 2771 * 41 ff 72 f8 pushq -0x8(%r10) 2772 * 55 push %rbp 2773 * 48 89 e5 mov %rsp,%rbp 2774 * (more pushes) 2775 * 41 52 push %r10 2776 * ... 2777 * 41 5a pop %r10 2778 * (more pops) 2779 * 5d pop %rbp 2780 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2781 * c3 retq 2782 * 2783 * There are some variations in the epilogues, like: 2784 * 2785 * 5b pop %rbx 2786 * 41 5a pop %r10 2787 * 41 5c pop %r12 2788 * 41 5d pop %r13 2789 * 41 5e pop %r14 2790 * c9 leaveq 2791 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2792 * c3 retq 2793 * 2794 * and: 2795 * 2796 * 4c 8b 55 e8 mov -0x18(%rbp),%r10 2797 * 48 8b 5d e0 mov -0x20(%rbp),%rbx 2798 * 4c 8b 65 f0 mov -0x10(%rbp),%r12 2799 * 4c 8b 6d f8 mov -0x8(%rbp),%r13 2800 * c9 leaveq 2801 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2802 * c3 retq 2803 * 2804 * Sometimes r13 is used as the DRAP register, in which case it's saved and 2805 * restored beforehand: 2806 * 2807 * 41 55 push %r13 2808 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13 2809 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp 2810 * ... 2811 * 49 8d 65 f0 lea -0x10(%r13),%rsp 2812 * 41 5d pop %r13 2813 * c3 retq 2814 */ 2815 static int update_cfi_state(struct instruction *insn, 2816 struct instruction *next_insn, 2817 struct cfi_state *cfi, struct stack_op *op) 2818 { 2819 struct cfi_reg *cfa = &cfi->cfa; 2820 struct cfi_reg *regs = cfi->regs; 2821 2822 /* ignore UNWIND_HINT_UNDEFINED regions */ 2823 if (cfi->force_undefined) 2824 return 0; 2825 2826 /* stack operations don't make sense with an undefined CFA */ 2827 if (cfa->base == CFI_UNDEFINED) { 2828 if (insn_func(insn)) { 2829 WARN_INSN(insn, "undefined stack state"); 2830 return 1; 2831 } 2832 return 0; 2833 } 2834 2835 if (cfi->type == UNWIND_HINT_TYPE_REGS || 2836 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) 2837 return update_cfi_state_regs(insn, cfi, op); 2838 2839 switch (op->dest.type) { 2840 2841 case OP_DEST_REG: 2842 switch (op->src.type) { 2843 2844 case OP_SRC_REG: 2845 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP && 2846 cfa->base == CFI_SP && 2847 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) { 2848 2849 /* mov %rsp, %rbp */ 2850 cfa->base = op->dest.reg; 2851 cfi->bp_scratch = false; 2852 } 2853 2854 else if (op->src.reg == CFI_SP && 2855 op->dest.reg == CFI_BP && cfi->drap) { 2856 2857 /* drap: mov %rsp, %rbp */ 2858 regs[CFI_BP].base = CFI_BP; 2859 regs[CFI_BP].offset = -cfi->stack_size; 2860 cfi->bp_scratch = false; 2861 } 2862 2863 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2864 2865 /* 2866 * mov %rsp, %reg 2867 * 2868 * This is needed for the rare case where GCC 2869 * does: 2870 * 2871 * mov %rsp, %rax 2872 * ... 2873 * mov %rax, %rsp 2874 */ 2875 cfi->vals[op->dest.reg].base = CFI_CFA; 2876 cfi->vals[op->dest.reg].offset = -cfi->stack_size; 2877 } 2878 2879 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP && 2880 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) { 2881 2882 /* 2883 * mov %rbp, %rsp 2884 * 2885 * Restore the original stack pointer (Clang). 2886 */ 2887 cfi->stack_size = -cfi->regs[CFI_BP].offset; 2888 } 2889 2890 else if (op->dest.reg == cfa->base) { 2891 2892 /* mov %reg, %rsp */ 2893 if (cfa->base == CFI_SP && 2894 cfi->vals[op->src.reg].base == CFI_CFA) { 2895 2896 /* 2897 * This is needed for the rare case 2898 * where GCC does something dumb like: 2899 * 2900 * lea 0x8(%rsp), %rcx 2901 * ... 2902 * mov %rcx, %rsp 2903 */ 2904 cfa->offset = -cfi->vals[op->src.reg].offset; 2905 cfi->stack_size = cfa->offset; 2906 2907 } else if (cfa->base == CFI_SP && 2908 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2909 cfi->vals[op->src.reg].offset == cfa->offset) { 2910 2911 /* 2912 * Stack swizzle: 2913 * 2914 * 1: mov %rsp, (%[tos]) 2915 * 2: mov %[tos], %rsp 2916 * ... 2917 * 3: pop %rsp 2918 * 2919 * Where: 2920 * 2921 * 1 - places a pointer to the previous 2922 * stack at the Top-of-Stack of the 2923 * new stack. 2924 * 2925 * 2 - switches to the new stack. 2926 * 2927 * 3 - pops the Top-of-Stack to restore 2928 * the original stack. 2929 * 2930 * Note: we set base to SP_INDIRECT 2931 * here and preserve offset. Therefore 2932 * when the unwinder reaches ToS it 2933 * will dereference SP and then add the 2934 * offset to find the next frame, IOW: 2935 * (%rsp) + offset. 2936 */ 2937 cfa->base = CFI_SP_INDIRECT; 2938 2939 } else { 2940 cfa->base = CFI_UNDEFINED; 2941 cfa->offset = 0; 2942 } 2943 } 2944 2945 else if (op->dest.reg == CFI_SP && 2946 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2947 cfi->vals[op->src.reg].offset == cfa->offset) { 2948 2949 /* 2950 * The same stack swizzle case 2) as above. But 2951 * because we can't change cfa->base, case 3) 2952 * will become a regular POP. Pretend we're a 2953 * PUSH so things don't go unbalanced. 2954 */ 2955 cfi->stack_size += 8; 2956 } 2957 2958 else if (cfi->vals[op->src.reg].base == CFI_CFA) { 2959 /* 2960 * Clang RSP musical chairs: 2961 * 2962 * mov %rsp, %rdx [handled above] 2963 * ... 2964 * mov %rdx, %rbx [handled here] 2965 * ... 2966 * mov %rbx, %rsp [handled above] 2967 */ 2968 cfi->vals[op->dest.reg].base = CFI_CFA; 2969 cfi->vals[op->dest.reg].offset = cfi->vals[op->src.reg].offset; 2970 } 2971 2972 2973 break; 2974 2975 case OP_SRC_ADD: 2976 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) { 2977 2978 /* add imm, %rsp */ 2979 cfi->stack_size -= op->src.offset; 2980 if (cfa->base == CFI_SP) 2981 cfa->offset -= op->src.offset; 2982 break; 2983 } 2984 2985 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP && 2986 insn->sym->frame_pointer) { 2987 /* addi.d fp,sp,imm on LoongArch */ 2988 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) { 2989 cfa->base = CFI_BP; 2990 cfa->offset = 0; 2991 } 2992 break; 2993 } 2994 2995 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { 2996 /* addi.d sp,fp,imm on LoongArch */ 2997 if (cfa->base == CFI_BP && cfa->offset == 0) { 2998 if (insn->sym->frame_pointer) { 2999 cfa->base = CFI_SP; 3000 cfa->offset = -op->src.offset; 3001 } 3002 } else { 3003 /* lea disp(%rbp), %rsp */ 3004 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); 3005 } 3006 break; 3007 } 3008 3009 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 3010 3011 /* drap: lea disp(%rsp), %drap */ 3012 cfi->drap_reg = op->dest.reg; 3013 3014 /* 3015 * lea disp(%rsp), %reg 3016 * 3017 * This is needed for the rare case where GCC 3018 * does something dumb like: 3019 * 3020 * lea 0x8(%rsp), %rcx 3021 * ... 3022 * mov %rcx, %rsp 3023 */ 3024 cfi->vals[op->dest.reg].base = CFI_CFA; 3025 cfi->vals[op->dest.reg].offset = \ 3026 -cfi->stack_size + op->src.offset; 3027 3028 break; 3029 } 3030 3031 if (cfi->drap && op->dest.reg == CFI_SP && 3032 op->src.reg == cfi->drap_reg) { 3033 3034 /* drap: lea disp(%drap), %rsp */ 3035 cfa->base = CFI_SP; 3036 cfa->offset = cfi->stack_size = -op->src.offset; 3037 cfi->drap_reg = CFI_UNDEFINED; 3038 cfi->drap = false; 3039 break; 3040 } 3041 3042 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { 3043 WARN_INSN(insn, "unsupported stack register modification"); 3044 return -1; 3045 } 3046 3047 break; 3048 3049 case OP_SRC_AND: 3050 if (op->dest.reg != CFI_SP || 3051 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || 3052 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { 3053 WARN_INSN(insn, "unsupported stack pointer realignment"); 3054 return -1; 3055 } 3056 3057 if (cfi->drap_reg != CFI_UNDEFINED) { 3058 /* drap: and imm, %rsp */ 3059 cfa->base = cfi->drap_reg; 3060 cfa->offset = cfi->stack_size = 0; 3061 cfi->drap = true; 3062 } 3063 3064 /* 3065 * Older versions of GCC (4.8ish) realign the stack 3066 * without DRAP, with a frame pointer. 3067 */ 3068 3069 break; 3070 3071 case OP_SRC_POP: 3072 case OP_SRC_POPF: 3073 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) { 3074 3075 /* pop %rsp; # restore from a stack swizzle */ 3076 cfa->base = CFI_SP; 3077 break; 3078 } 3079 3080 if (!cfi->drap && op->dest.reg == cfa->base) { 3081 3082 /* pop %rbp */ 3083 cfa->base = CFI_SP; 3084 } 3085 3086 if (cfi->drap && cfa->base == CFI_BP_INDIRECT && 3087 op->dest.reg == cfi->drap_reg && 3088 cfi->drap_offset == -cfi->stack_size) { 3089 3090 /* drap: pop %drap */ 3091 cfa->base = cfi->drap_reg; 3092 cfa->offset = 0; 3093 cfi->drap_offset = -1; 3094 3095 } else if (cfi->stack_size == -regs[op->dest.reg].offset) { 3096 3097 /* pop %reg */ 3098 restore_reg(cfi, op->dest.reg); 3099 } 3100 3101 cfi->stack_size -= 8; 3102 if (cfa->base == CFI_SP) 3103 cfa->offset -= 8; 3104 3105 break; 3106 3107 case OP_SRC_REG_INDIRECT: 3108 if (!cfi->drap && op->dest.reg == cfa->base && 3109 op->dest.reg == CFI_BP) { 3110 3111 /* mov disp(%rsp), %rbp */ 3112 cfa->base = CFI_SP; 3113 cfa->offset = cfi->stack_size; 3114 } 3115 3116 if (cfi->drap && op->src.reg == CFI_BP && 3117 op->src.offset == cfi->drap_offset) { 3118 3119 /* drap: mov disp(%rbp), %drap */ 3120 cfa->base = cfi->drap_reg; 3121 cfa->offset = 0; 3122 cfi->drap_offset = -1; 3123 } 3124 3125 if (cfi->drap && op->src.reg == CFI_BP && 3126 op->src.offset == regs[op->dest.reg].offset) { 3127 3128 /* drap: mov disp(%rbp), %reg */ 3129 restore_reg(cfi, op->dest.reg); 3130 3131 } else if (op->src.reg == cfa->base && 3132 op->src.offset == regs[op->dest.reg].offset + cfa->offset) { 3133 3134 /* mov disp(%rbp), %reg */ 3135 /* mov disp(%rsp), %reg */ 3136 restore_reg(cfi, op->dest.reg); 3137 3138 } else if (op->src.reg == CFI_SP && 3139 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) { 3140 3141 /* mov disp(%rsp), %reg */ 3142 restore_reg(cfi, op->dest.reg); 3143 } 3144 3145 break; 3146 3147 default: 3148 WARN_INSN(insn, "unknown stack-related instruction"); 3149 return -1; 3150 } 3151 3152 break; 3153 3154 case OP_DEST_PUSH: 3155 case OP_DEST_PUSHF: 3156 cfi->stack_size += 8; 3157 if (cfa->base == CFI_SP) 3158 cfa->offset += 8; 3159 3160 if (op->src.type != OP_SRC_REG) 3161 break; 3162 3163 if (cfi->drap) { 3164 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3165 3166 /* drap: push %drap */ 3167 cfa->base = CFI_BP_INDIRECT; 3168 cfa->offset = -cfi->stack_size; 3169 3170 /* save drap so we know when to restore it */ 3171 cfi->drap_offset = -cfi->stack_size; 3172 3173 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) { 3174 3175 /* drap: push %rbp */ 3176 cfi->stack_size = 0; 3177 3178 } else { 3179 3180 /* drap: push %reg */ 3181 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); 3182 } 3183 3184 } else { 3185 3186 /* push %reg */ 3187 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size); 3188 } 3189 3190 /* detect when asm code uses rbp as a scratch register */ 3191 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP && 3192 cfa->base != CFI_BP) 3193 cfi->bp_scratch = true; 3194 break; 3195 3196 case OP_DEST_REG_INDIRECT: 3197 3198 if (cfi->drap) { 3199 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3200 3201 /* drap: mov %drap, disp(%rbp) */ 3202 cfa->base = CFI_BP_INDIRECT; 3203 cfa->offset = op->dest.offset; 3204 3205 /* save drap offset so we know when to restore it */ 3206 cfi->drap_offset = op->dest.offset; 3207 } else { 3208 3209 /* drap: mov reg, disp(%rbp) */ 3210 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); 3211 } 3212 3213 } else if (op->dest.reg == cfa->base) { 3214 3215 /* mov reg, disp(%rbp) */ 3216 /* mov reg, disp(%rsp) */ 3217 save_reg(cfi, op->src.reg, CFI_CFA, 3218 op->dest.offset - cfi->cfa.offset); 3219 3220 } else if (op->dest.reg == CFI_SP) { 3221 3222 /* mov reg, disp(%rsp) */ 3223 save_reg(cfi, op->src.reg, CFI_CFA, 3224 op->dest.offset - cfi->stack_size); 3225 3226 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) { 3227 3228 /* mov %rsp, (%reg); # setup a stack swizzle. */ 3229 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT; 3230 cfi->vals[op->dest.reg].offset = cfa->offset; 3231 } 3232 3233 break; 3234 3235 case OP_DEST_MEM: 3236 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { 3237 WARN_INSN(insn, "unknown stack-related memory operation"); 3238 return -1; 3239 } 3240 3241 /* pop mem */ 3242 cfi->stack_size -= 8; 3243 if (cfa->base == CFI_SP) 3244 cfa->offset -= 8; 3245 3246 break; 3247 3248 default: 3249 WARN_INSN(insn, "unknown stack-related instruction"); 3250 return -1; 3251 } 3252 3253 return 0; 3254 } 3255 3256 /* 3257 * The stack layouts of alternatives instructions can sometimes diverge when 3258 * they have stack modifications. That's fine as long as the potential stack 3259 * layouts don't conflict at any given potential instruction boundary. 3260 * 3261 * Flatten the CFIs of the different alternative code streams (both original 3262 * and replacement) into a single shared CFI array which can be used to detect 3263 * conflicts and nicely feed a linear array of ORC entries to the unwinder. 3264 */ 3265 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn) 3266 { 3267 struct cfi_state **alt_cfi; 3268 int group_off; 3269 3270 if (!insn->alt_group) 3271 return 0; 3272 3273 if (!insn->cfi) { 3274 WARN("CFI missing"); 3275 return -1; 3276 } 3277 3278 alt_cfi = insn->alt_group->cfi; 3279 group_off = insn->offset - insn->alt_group->first_insn->offset; 3280 3281 if (!alt_cfi[group_off]) { 3282 alt_cfi[group_off] = insn->cfi; 3283 } else { 3284 if (cficmp(alt_cfi[group_off], insn->cfi)) { 3285 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group; 3286 struct instruction *orig = orig_group->first_insn; 3287 WARN_INSN(orig, "stack layout conflict in alternatives: %s", 3288 offstr(insn->sec, insn->offset)); 3289 return -1; 3290 } 3291 } 3292 3293 return 0; 3294 } 3295 3296 static int noinline handle_insn_ops(struct instruction *insn, 3297 struct instruction *next_insn, 3298 struct insn_state *state) 3299 { 3300 struct insn_state prev_state __maybe_unused = *state; 3301 struct stack_op *op; 3302 int ret = 0; 3303 3304 for (op = insn->stack_ops; op; op = op->next) { 3305 3306 ret = update_cfi_state(insn, next_insn, &state->cfi, op); 3307 if (ret) 3308 goto done; 3309 3310 if (!opts.uaccess || !insn->alt_group) 3311 continue; 3312 3313 if (op->dest.type == OP_DEST_PUSHF) { 3314 if (!state->uaccess_stack) { 3315 state->uaccess_stack = 1; 3316 } else if (state->uaccess_stack >> 31) { 3317 WARN_INSN(insn, "PUSHF stack exhausted"); 3318 ret = 1; 3319 goto done; 3320 } 3321 state->uaccess_stack <<= 1; 3322 state->uaccess_stack |= state->uaccess; 3323 } 3324 3325 if (op->src.type == OP_SRC_POPF) { 3326 if (state->uaccess_stack) { 3327 state->uaccess = state->uaccess_stack & 1; 3328 state->uaccess_stack >>= 1; 3329 if (state->uaccess_stack == 1) 3330 state->uaccess_stack = 0; 3331 } 3332 } 3333 } 3334 3335 done: 3336 TRACE_INSN_STATE(insn, &prev_state, state); 3337 3338 return ret; 3339 } 3340 3341 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) 3342 { 3343 struct cfi_state *cfi1 = insn->cfi; 3344 int i; 3345 3346 if (!cfi1) { 3347 WARN("CFI missing"); 3348 return false; 3349 } 3350 3351 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { 3352 3353 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d", 3354 cfi1->cfa.base, cfi1->cfa.offset, 3355 cfi2->cfa.base, cfi2->cfa.offset); 3356 return false; 3357 3358 } 3359 3360 if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) { 3361 for (i = 0; i < CFI_NUM_REGS; i++) { 3362 3363 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], sizeof(struct cfi_reg))) 3364 continue; 3365 3366 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", 3367 i, cfi1->regs[i].base, cfi1->regs[i].offset, 3368 i, cfi2->regs[i].base, cfi2->regs[i].offset); 3369 } 3370 return false; 3371 } 3372 3373 if (cfi1->type != cfi2->type) { 3374 3375 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d", 3376 cfi1->type, cfi2->type); 3377 return false; 3378 } 3379 3380 if (cfi1->drap != cfi2->drap || 3381 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || 3382 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { 3383 3384 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", 3385 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, 3386 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); 3387 return false; 3388 } 3389 3390 return true; 3391 } 3392 3393 static inline bool func_uaccess_safe(struct symbol *func) 3394 { 3395 if (func) 3396 return func->uaccess_safe; 3397 3398 return false; 3399 } 3400 3401 static inline const char *call_dest_name(struct instruction *insn) 3402 { 3403 static char pvname[19]; 3404 struct reloc *reloc; 3405 int idx; 3406 3407 if (insn_call_dest(insn)) 3408 return insn_call_dest(insn)->name; 3409 3410 reloc = insn_reloc(NULL, insn); 3411 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) { 3412 idx = (reloc_addend(reloc) / sizeof(void *)); 3413 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx); 3414 return pvname; 3415 } 3416 3417 return "{dynamic}"; 3418 } 3419 3420 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn) 3421 { 3422 struct symbol *target; 3423 struct reloc *reloc; 3424 int idx; 3425 3426 reloc = insn_reloc(file, insn); 3427 if (!reloc || strcmp(reloc->sym->name, "pv_ops")) 3428 return false; 3429 3430 idx = arch_insn_adjusted_addend(insn, reloc) / sizeof(void *); 3431 3432 if (file->pv_ops[idx].clean) 3433 return true; 3434 3435 file->pv_ops[idx].clean = true; 3436 3437 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) { 3438 if (!target->sec->noinstr) { 3439 WARN("pv_ops[%d]: %s", idx, target->name); 3440 file->pv_ops[idx].clean = false; 3441 } 3442 } 3443 3444 return file->pv_ops[idx].clean; 3445 } 3446 3447 static inline bool noinstr_call_dest(struct objtool_file *file, 3448 struct instruction *insn, 3449 struct symbol *func) 3450 { 3451 /* 3452 * We can't deal with indirect function calls at present; 3453 * assume they're instrumented. 3454 */ 3455 if (!func) { 3456 if (file->pv_ops) 3457 return pv_call_dest(file, insn); 3458 3459 return false; 3460 } 3461 3462 /* 3463 * If the symbol is from a noinstr section; we good. 3464 */ 3465 if (func->sec->noinstr) 3466 return true; 3467 3468 /* 3469 * If the symbol is a static_call trampoline, we can't tell. 3470 */ 3471 if (func->static_call_tramp) 3472 return true; 3473 3474 /* 3475 * The __ubsan_handle_*() calls are like WARN(), they only happen when 3476 * something 'BAD' happened. At the risk of taking the machine down, 3477 * let them proceed to get the message out. 3478 */ 3479 if (!strncmp(func->name, "__ubsan_handle_", 15)) 3480 return true; 3481 3482 return false; 3483 } 3484 3485 static int validate_call(struct objtool_file *file, 3486 struct instruction *insn, 3487 struct insn_state *state) 3488 { 3489 if (state->noinstr && state->instr <= 0 && 3490 !noinstr_call_dest(file, insn, insn_call_dest(insn))) { 3491 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn)); 3492 return 1; 3493 } 3494 3495 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) { 3496 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn)); 3497 return 1; 3498 } 3499 3500 if (state->df) { 3501 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn)); 3502 return 1; 3503 } 3504 3505 return 0; 3506 } 3507 3508 static int validate_sibling_call(struct objtool_file *file, 3509 struct instruction *insn, 3510 struct insn_state *state) 3511 { 3512 if (insn_func(insn) && has_modified_stack_frame(insn, state)) { 3513 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame"); 3514 return 1; 3515 } 3516 3517 return validate_call(file, insn, state); 3518 } 3519 3520 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 3521 { 3522 if (state->noinstr && state->instr > 0) { 3523 WARN_INSN(insn, "return with instrumentation enabled"); 3524 return 1; 3525 } 3526 3527 if (state->uaccess && !func_uaccess_safe(func)) { 3528 WARN_INSN(insn, "return with UACCESS enabled"); 3529 return 1; 3530 } 3531 3532 if (!state->uaccess && func_uaccess_safe(func)) { 3533 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function"); 3534 return 1; 3535 } 3536 3537 if (state->df) { 3538 WARN_INSN(insn, "return with DF set"); 3539 return 1; 3540 } 3541 3542 if (func && has_modified_stack_frame(insn, state)) { 3543 WARN_INSN(insn, "return with modified stack frame"); 3544 return 1; 3545 } 3546 3547 if (state->cfi.bp_scratch) { 3548 WARN_INSN(insn, "BP used as a scratch register"); 3549 return 1; 3550 } 3551 3552 return 0; 3553 } 3554 3555 static struct instruction *next_insn_to_validate(struct objtool_file *file, 3556 struct instruction *insn) 3557 { 3558 struct alt_group *alt_group = insn->alt_group; 3559 3560 /* 3561 * Simulate the fact that alternatives are patched in-place. When the 3562 * end of a replacement alt_group is reached, redirect objtool flow to 3563 * the end of the original alt_group. 3564 * 3565 * insn->alts->insn -> alt_group->first_insn 3566 * ... 3567 * alt_group->last_insn 3568 * [alt_group->nop] -> next(orig_group->last_insn) 3569 */ 3570 if (alt_group) { 3571 if (alt_group->nop) { 3572 /* ->nop implies ->orig_group */ 3573 if (insn == alt_group->last_insn) 3574 return alt_group->nop; 3575 if (insn == alt_group->nop) 3576 goto next_orig; 3577 } 3578 if (insn == alt_group->last_insn && alt_group->orig_group) 3579 goto next_orig; 3580 } 3581 3582 return next_insn_same_sec(file, insn); 3583 3584 next_orig: 3585 return next_insn_same_sec(file, alt_group->orig_group->last_insn); 3586 } 3587 3588 static bool skip_alt_group(struct instruction *insn) 3589 { 3590 struct instruction *alt_insn = insn->alts ? insn->alts->insn : NULL; 3591 3592 if (!insn->alt_group) 3593 return false; 3594 3595 /* ANNOTATE_IGNORE_ALTERNATIVE */ 3596 if (insn->alt_group->ignore) { 3597 TRACE_ALT(insn, "alt group ignored"); 3598 return true; 3599 } 3600 3601 /* 3602 * For NOP patched with CLAC/STAC, only follow the latter to avoid 3603 * impossible code paths combining patched CLAC with unpatched STAC 3604 * or vice versa. 3605 * 3606 * ANNOTATE_IGNORE_ALTERNATIVE could have been used here, but Linus 3607 * requested not to do that to avoid hurting .s file readability 3608 * around CLAC/STAC alternative sites. 3609 */ 3610 3611 if (!alt_insn) 3612 return false; 3613 3614 /* Don't override ASM_{CLAC,STAC}_UNSAFE */ 3615 if (alt_insn->alt_group && alt_insn->alt_group->ignore) 3616 return false; 3617 3618 return alt_insn->type == INSN_CLAC || alt_insn->type == INSN_STAC; 3619 } 3620 3621 static int validate_branch(struct objtool_file *file, struct symbol *func, 3622 struct instruction *insn, struct insn_state state); 3623 static int do_validate_branch(struct objtool_file *file, struct symbol *func, 3624 struct instruction *insn, struct insn_state *state); 3625 3626 static int validate_insn(struct objtool_file *file, struct symbol *func, 3627 struct instruction *insn, struct insn_state *statep, 3628 struct instruction *prev_insn, struct instruction *next_insn, 3629 bool *dead_end) 3630 { 3631 char *alt_name __maybe_unused = NULL; 3632 struct alternative *alt; 3633 u8 visited; 3634 int ret; 3635 3636 /* 3637 * Any returns before the end of this function are effectively dead 3638 * ends, i.e. validate_branch() has reached the end of the branch. 3639 */ 3640 *dead_end = true; 3641 3642 visited = VISITED_BRANCH << statep->uaccess; 3643 if (insn->visited & VISITED_BRANCH_MASK) { 3644 if (!insn->hint && !insn_cfi_match(insn, &statep->cfi)) 3645 return 1; 3646 3647 if (insn->visited & visited) { 3648 TRACE_INSN(insn, "already visited"); 3649 return 0; 3650 } 3651 } else { 3652 nr_insns_visited++; 3653 } 3654 3655 if (statep->noinstr) 3656 statep->instr += insn->instr; 3657 3658 if (insn->hint) { 3659 if (insn->restore) { 3660 struct instruction *save_insn, *i; 3661 3662 i = insn; 3663 save_insn = NULL; 3664 3665 sym_for_each_insn_continue_reverse(file, func, i) { 3666 if (i->save) { 3667 save_insn = i; 3668 break; 3669 } 3670 } 3671 3672 if (!save_insn) { 3673 WARN_INSN(insn, "no corresponding CFI save for CFI restore"); 3674 return 1; 3675 } 3676 3677 if (!save_insn->visited) { 3678 /* 3679 * If the restore hint insn is at the 3680 * beginning of a basic block and was 3681 * branched to from elsewhere, and the 3682 * save insn hasn't been visited yet, 3683 * defer following this branch for now. 3684 * It will be seen later via the 3685 * straight-line path. 3686 */ 3687 if (!prev_insn) { 3688 TRACE_INSN(insn, "defer restore"); 3689 return 0; 3690 } 3691 3692 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo"); 3693 return 1; 3694 } 3695 3696 insn->cfi = save_insn->cfi; 3697 nr_cfi_reused++; 3698 } 3699 3700 statep->cfi = *insn->cfi; 3701 } else { 3702 /* XXX track if we actually changed statep->cfi */ 3703 3704 if (prev_insn && !cficmp(prev_insn->cfi, &statep->cfi)) { 3705 insn->cfi = prev_insn->cfi; 3706 nr_cfi_reused++; 3707 } else { 3708 insn->cfi = cfi_hash_find_or_add(&statep->cfi); 3709 } 3710 } 3711 3712 insn->visited |= visited; 3713 3714 if (propagate_alt_cfi(file, insn)) 3715 return 1; 3716 3717 if (insn->alts) { 3718 for (alt = insn->alts; alt; alt = alt->next) { 3719 TRACE_ALT_BEGIN(insn, alt, alt_name); 3720 ret = validate_branch(file, func, alt->insn, *statep); 3721 TRACE_ALT_END(insn, alt, alt_name); 3722 if (ret) { 3723 BT_INSN(insn, "(alt)"); 3724 return ret; 3725 } 3726 } 3727 TRACE_ALT_INFO_NOADDR(insn, "/ ", "DEFAULT"); 3728 } 3729 3730 if (skip_alt_group(insn)) 3731 return 0; 3732 3733 if (handle_insn_ops(insn, next_insn, statep)) 3734 return 1; 3735 3736 switch (insn->type) { 3737 3738 case INSN_RETURN: 3739 TRACE_INSN(insn, "return"); 3740 return validate_return(func, insn, statep); 3741 3742 case INSN_CALL: 3743 case INSN_CALL_DYNAMIC: 3744 if (insn->type == INSN_CALL) 3745 TRACE_INSN(insn, "call"); 3746 else 3747 TRACE_INSN(insn, "indirect call"); 3748 3749 ret = validate_call(file, insn, statep); 3750 if (ret) 3751 return ret; 3752 3753 if (opts.stackval && func && !is_special_call(insn) && 3754 !has_valid_stack_frame(statep)) { 3755 WARN_INSN(insn, "call without frame pointer save/setup"); 3756 return 1; 3757 } 3758 3759 break; 3760 3761 case INSN_JUMP_CONDITIONAL: 3762 case INSN_JUMP_UNCONDITIONAL: 3763 if (is_sibling_call(insn)) { 3764 TRACE_INSN(insn, "sibling call"); 3765 ret = validate_sibling_call(file, insn, statep); 3766 if (ret) 3767 return ret; 3768 3769 } else if (insn->jump_dest) { 3770 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3771 TRACE_INSN(insn, "unconditional jump"); 3772 else 3773 TRACE_INSN(insn, "jump taken"); 3774 3775 ret = validate_branch(file, func, insn->jump_dest, *statep); 3776 if (ret) { 3777 BT_INSN(insn, "(branch)"); 3778 return ret; 3779 } 3780 } 3781 3782 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3783 return 0; 3784 3785 TRACE_INSN(insn, "jump not taken"); 3786 break; 3787 3788 case INSN_JUMP_DYNAMIC: 3789 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3790 TRACE_INSN(insn, "indirect jump"); 3791 if (is_sibling_call(insn)) { 3792 ret = validate_sibling_call(file, insn, statep); 3793 if (ret) 3794 return ret; 3795 } 3796 3797 if (insn->type == INSN_JUMP_DYNAMIC) 3798 return 0; 3799 3800 break; 3801 3802 case INSN_SYSCALL: 3803 TRACE_INSN(insn, "syscall"); 3804 if (func && (!next_insn || !next_insn->hint)) { 3805 WARN_INSN(insn, "unsupported instruction in callable function"); 3806 return 1; 3807 } 3808 3809 break; 3810 3811 case INSN_SYSRET: 3812 TRACE_INSN(insn, "sysret"); 3813 if (func && (!next_insn || !next_insn->hint)) { 3814 WARN_INSN(insn, "unsupported instruction in callable function"); 3815 return 1; 3816 } 3817 3818 return 0; 3819 3820 case INSN_STAC: 3821 TRACE_INSN(insn, "stac"); 3822 if (!opts.uaccess) 3823 break; 3824 3825 if (statep->uaccess) { 3826 WARN_INSN(insn, "recursive UACCESS enable"); 3827 return 1; 3828 } 3829 3830 statep->uaccess = true; 3831 break; 3832 3833 case INSN_CLAC: 3834 TRACE_INSN(insn, "clac"); 3835 if (!opts.uaccess) 3836 break; 3837 3838 if (!statep->uaccess && func) { 3839 WARN_INSN(insn, "redundant UACCESS disable"); 3840 return 1; 3841 } 3842 3843 if (func_uaccess_safe(func) && !statep->uaccess_stack) { 3844 WARN_INSN(insn, "UACCESS-safe disables UACCESS"); 3845 return 1; 3846 } 3847 3848 statep->uaccess = false; 3849 break; 3850 3851 case INSN_STD: 3852 TRACE_INSN(insn, "std"); 3853 if (statep->df) { 3854 WARN_INSN(insn, "recursive STD"); 3855 return 1; 3856 } 3857 3858 statep->df = true; 3859 break; 3860 3861 case INSN_CLD: 3862 TRACE_INSN(insn, "cld"); 3863 if (!statep->df && func) { 3864 WARN_INSN(insn, "redundant CLD"); 3865 return 1; 3866 } 3867 3868 statep->df = false; 3869 break; 3870 3871 default: 3872 break; 3873 } 3874 3875 if (insn->dead_end) 3876 TRACE_INSN(insn, "dead end"); 3877 3878 *dead_end = insn->dead_end; 3879 return 0; 3880 } 3881 3882 /* 3883 * Follow the branch starting at the given instruction, and recursively follow 3884 * any other branches (jumps). Meanwhile, track the frame pointer state at 3885 * each instruction and validate all the rules described in 3886 * tools/objtool/Documentation/objtool.txt. 3887 */ 3888 static int do_validate_branch(struct objtool_file *file, struct symbol *func, 3889 struct instruction *insn, struct insn_state *state) 3890 { 3891 struct instruction *next_insn, *prev_insn = NULL; 3892 bool dead_end; 3893 int ret; 3894 3895 if (func && func->ignore) 3896 return 0; 3897 3898 do { 3899 insn->trace = 0; 3900 next_insn = next_insn_to_validate(file, insn); 3901 3902 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { 3903 /* Ignore KCFI type preambles, which always fall through */ 3904 if (is_prefix_func(func)) 3905 return 0; 3906 3907 if (file->ignore_unreachables) 3908 return 0; 3909 3910 WARN("%s() falls through to next function %s()", 3911 func->name, insn_func(insn)->name); 3912 func->warned = 1; 3913 3914 return 1; 3915 } 3916 3917 ret = validate_insn(file, func, insn, state, prev_insn, next_insn, 3918 &dead_end); 3919 3920 if (!insn->trace) { 3921 if (ret) 3922 TRACE_INSN(insn, "warning (%d)", ret); 3923 else 3924 TRACE_INSN(insn, NULL); 3925 } 3926 3927 if (!dead_end && !next_insn) { 3928 if (state->cfi.cfa.base == CFI_UNDEFINED) 3929 return 0; 3930 if (file->ignore_unreachables) 3931 return 0; 3932 3933 WARN("%s%sunexpected end of section %s", 3934 func ? func->name : "", func ? "(): " : "", 3935 insn->sec->name); 3936 return 1; 3937 } 3938 3939 prev_insn = insn; 3940 insn = next_insn; 3941 3942 } while (!dead_end); 3943 3944 return ret; 3945 } 3946 3947 static int validate_branch(struct objtool_file *file, struct symbol *func, 3948 struct instruction *insn, struct insn_state state) 3949 { 3950 int ret; 3951 3952 trace_depth_inc(); 3953 ret = do_validate_branch(file, func, insn, &state); 3954 trace_depth_dec(); 3955 3956 return ret; 3957 } 3958 3959 static int validate_unwind_hint(struct objtool_file *file, 3960 struct instruction *insn, 3961 struct insn_state *state) 3962 { 3963 if (insn->hint && !insn->visited) { 3964 struct symbol *func = insn_func(insn); 3965 int ret; 3966 3967 ret = validate_branch(file, func, insn, *state); 3968 if (ret) 3969 BT_INSN(insn, "<=== (hint)"); 3970 return ret; 3971 } 3972 3973 return 0; 3974 } 3975 3976 static int validate_unwind_hints(struct objtool_file *file, struct section *sec) 3977 { 3978 struct instruction *insn; 3979 struct insn_state state; 3980 int warnings = 0; 3981 3982 if (!file->hints) 3983 return 0; 3984 3985 init_insn_state(file, &state, sec); 3986 3987 if (sec) { 3988 sec_for_each_insn(file, sec, insn) 3989 warnings += validate_unwind_hint(file, insn, &state); 3990 } else { 3991 for_each_insn(file, insn) 3992 warnings += validate_unwind_hint(file, insn, &state); 3993 } 3994 3995 return warnings; 3996 } 3997 3998 /* 3999 * Validate rethunk entry constraint: must untrain RET before the first RET. 4000 * 4001 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes 4002 * before an actual RET instruction. 4003 */ 4004 static int validate_unret(struct objtool_file *file, struct instruction *insn) 4005 { 4006 struct instruction *next, *dest; 4007 int ret; 4008 4009 for (;;) { 4010 next = next_insn_to_validate(file, insn); 4011 4012 if (insn->visited & VISITED_UNRET) 4013 return 0; 4014 4015 insn->visited |= VISITED_UNRET; 4016 4017 if (insn->alts) { 4018 struct alternative *alt; 4019 for (alt = insn->alts; alt; alt = alt->next) { 4020 ret = validate_unret(file, alt->insn); 4021 if (ret) { 4022 BT_INSN(insn, "(alt)"); 4023 return ret; 4024 } 4025 } 4026 } 4027 4028 switch (insn->type) { 4029 4030 case INSN_CALL_DYNAMIC: 4031 case INSN_JUMP_DYNAMIC: 4032 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4033 WARN_INSN(insn, "early indirect call"); 4034 return 1; 4035 4036 case INSN_JUMP_UNCONDITIONAL: 4037 case INSN_JUMP_CONDITIONAL: 4038 if (!is_sibling_call(insn)) { 4039 if (!insn->jump_dest) { 4040 WARN_INSN(insn, "unresolved jump target after linking?!?"); 4041 return 1; 4042 } 4043 ret = validate_unret(file, insn->jump_dest); 4044 if (ret) { 4045 BT_INSN(insn, "(branch%s)", 4046 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : ""); 4047 return ret; 4048 } 4049 4050 if (insn->type == INSN_JUMP_UNCONDITIONAL) 4051 return 0; 4052 4053 break; 4054 } 4055 4056 /* fallthrough */ 4057 case INSN_CALL: 4058 dest = find_insn(file, insn_call_dest(insn)->sec, 4059 insn_call_dest(insn)->offset); 4060 if (!dest) { 4061 WARN("Unresolved function after linking!?: %s", 4062 insn_call_dest(insn)->name); 4063 return 1; 4064 } 4065 4066 ret = validate_unret(file, dest); 4067 if (ret) { 4068 BT_INSN(insn, "(call)"); 4069 return ret; 4070 } 4071 /* 4072 * If a call returns without error, it must have seen UNTRAIN_RET. 4073 * Therefore any non-error return is a success. 4074 */ 4075 return 0; 4076 4077 case INSN_RETURN: 4078 WARN_INSN(insn, "RET before UNTRAIN"); 4079 return 1; 4080 4081 case INSN_SYSCALL: 4082 break; 4083 4084 case INSN_SYSRET: 4085 return 0; 4086 4087 case INSN_NOP: 4088 if (insn->retpoline_safe) 4089 return 0; 4090 break; 4091 4092 default: 4093 break; 4094 } 4095 4096 if (insn->dead_end) 4097 return 0; 4098 4099 if (!next) { 4100 WARN_INSN(insn, "teh end!"); 4101 return 1; 4102 } 4103 insn = next; 4104 } 4105 4106 return 0; 4107 } 4108 4109 /* 4110 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter 4111 * VALIDATE_UNRET_END before RET. 4112 */ 4113 static int validate_unrets(struct objtool_file *file) 4114 { 4115 struct instruction *insn; 4116 int warnings = 0; 4117 4118 for_each_insn(file, insn) { 4119 if (!insn->unret) 4120 continue; 4121 4122 warnings += validate_unret(file, insn); 4123 } 4124 4125 return warnings; 4126 } 4127 4128 static int validate_retpoline(struct objtool_file *file) 4129 { 4130 struct instruction *insn; 4131 int warnings = 0; 4132 4133 for_each_insn(file, insn) { 4134 if (insn->type != INSN_JUMP_DYNAMIC && 4135 insn->type != INSN_CALL_DYNAMIC && 4136 insn->type != INSN_RETURN) 4137 continue; 4138 4139 if (insn->retpoline_safe) 4140 continue; 4141 4142 if (insn->sec->init) 4143 continue; 4144 4145 if (insn->type == INSN_RETURN) { 4146 if (opts.rethunk) { 4147 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build"); 4148 warnings++; 4149 } 4150 continue; 4151 } 4152 4153 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build", 4154 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); 4155 warnings++; 4156 } 4157 4158 if (!opts.cfi) 4159 return warnings; 4160 4161 /* 4162 * kCFI call sites look like: 4163 * 4164 * movl $(-0x12345678), %r10d 4165 * addl -4(%r11), %r10d 4166 * jz 1f 4167 * ud2 4168 * 1: cs call __x86_indirect_thunk_r11 4169 * 4170 * Verify all indirect calls are kCFI adorned by checking for the 4171 * UD2. Notably, doing __nocfi calls to regular (cfi) functions is 4172 * broken. 4173 */ 4174 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 4175 struct symbol *sym = insn->sym; 4176 4177 if (sym && (is_notype_sym(sym) || 4178 is_func_sym(sym)) && !sym->nocfi) { 4179 struct instruction *prev = 4180 prev_insn_same_sym(file, insn); 4181 4182 if (!prev || prev->type != INSN_BUG) { 4183 WARN_INSN(insn, "no-cfi indirect call!"); 4184 warnings++; 4185 } 4186 } 4187 } 4188 4189 return warnings; 4190 } 4191 4192 static bool is_kasan_insn(struct instruction *insn) 4193 { 4194 return (insn->type == INSN_CALL && 4195 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return")); 4196 } 4197 4198 static bool is_ubsan_insn(struct instruction *insn) 4199 { 4200 return (insn->type == INSN_CALL && 4201 !strcmp(insn_call_dest(insn)->name, 4202 "__ubsan_handle_builtin_unreachable")); 4203 } 4204 4205 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) 4206 { 4207 struct symbol *func = insn_func(insn); 4208 struct instruction *prev_insn; 4209 int i; 4210 4211 if (insn->type == INSN_NOP || insn->type == INSN_TRAP || 4212 insn->hole || (func && func->ignore)) 4213 return true; 4214 4215 /* 4216 * Ignore alternative replacement instructions. This can happen 4217 * when a whitelisted function uses one of the ALTERNATIVE macros. 4218 */ 4219 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 4220 !strcmp(insn->sec->name, ".altinstr_aux")) 4221 return true; 4222 4223 if (!func) 4224 return false; 4225 4226 if (func->static_call_tramp) 4227 return true; 4228 4229 /* 4230 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees 4231 * __builtin_unreachable(). The BUG() macro has an unreachable() after 4232 * the UD2, which causes GCC's undefined trap logic to emit another UD2 4233 * (or occasionally a JMP to UD2). 4234 * 4235 * It may also insert a UD2 after calling a __noreturn function. 4236 */ 4237 prev_insn = prev_insn_same_sec(file, insn); 4238 if (prev_insn && prev_insn->dead_end && 4239 (insn->type == INSN_BUG || 4240 (insn->type == INSN_JUMP_UNCONDITIONAL && 4241 insn->jump_dest && insn->jump_dest->type == INSN_BUG))) 4242 return true; 4243 4244 /* 4245 * Check if this (or a subsequent) instruction is related to 4246 * CONFIG_UBSAN or CONFIG_KASAN. 4247 * 4248 * End the search at 5 instructions to avoid going into the weeds. 4249 */ 4250 for (i = 0; i < 5; i++) { 4251 4252 if (is_kasan_insn(insn) || is_ubsan_insn(insn)) 4253 return true; 4254 4255 if (insn->type == INSN_JUMP_UNCONDITIONAL) { 4256 if (insn->jump_dest && 4257 insn_func(insn->jump_dest) == func) { 4258 insn = insn->jump_dest; 4259 continue; 4260 } 4261 4262 break; 4263 } 4264 4265 if (insn->offset + insn->len >= func->offset + func->len) 4266 break; 4267 4268 insn = next_insn_same_sec(file, insn); 4269 } 4270 4271 return false; 4272 } 4273 4274 /* 4275 * For FineIBT or kCFI, a certain number of bytes preceding the function may be 4276 * NOPs. Those NOPs may be rewritten at runtime and executed, so give them a 4277 * proper function name: __pfx_<func>. 4278 * 4279 * The NOPs may not exist for the following cases: 4280 * 4281 * - compiler cloned functions (*.cold, *.part0, etc) 4282 * - asm functions created with inline asm or without SYM_FUNC_START() 4283 * 4284 * Also, the function may already have a prefix from a previous objtool run 4285 * (livepatch extracted functions, or manually running objtool multiple times). 4286 * 4287 * So return 0 if the NOPs are missing or the function already has a prefix 4288 * symbol. 4289 */ 4290 static int create_prefix_symbol(struct objtool_file *file, struct symbol *func) 4291 { 4292 struct instruction *insn, *prev; 4293 char name[SYM_NAME_LEN]; 4294 struct cfi_state *cfi; 4295 4296 if (!is_func_sym(func) || is_prefix_func(func) || is_cold_func(func) || 4297 func->static_call_tramp) 4298 return 0; 4299 4300 if ((strlen(func->name) + sizeof("__pfx_") > SYM_NAME_LEN)) { 4301 WARN("%s: symbol name too long, can't create __pfx_ symbol", 4302 func->name); 4303 return 0; 4304 } 4305 4306 if (snprintf_check(name, SYM_NAME_LEN, "__pfx_%s", func->name)) 4307 return -1; 4308 4309 if (file->klp) { 4310 struct symbol *pfx; 4311 4312 pfx = find_symbol_by_offset(func->sec, func->offset - opts.prefix); 4313 if (pfx && is_prefix_func(pfx) && !strcmp(pfx->name, name)) 4314 return 0; 4315 } 4316 4317 insn = find_insn(file, func->sec, func->offset); 4318 if (!insn) { 4319 WARN("%s: can't find starting instruction", func->name); 4320 return -1; 4321 } 4322 4323 for (prev = prev_insn_same_sec(file, insn); 4324 prev; 4325 prev = prev_insn_same_sec(file, prev)) { 4326 u64 offset; 4327 4328 if (prev->type != INSN_NOP) 4329 return 0; 4330 4331 offset = func->offset - prev->offset; 4332 4333 if (offset > opts.prefix) 4334 return 0; 4335 4336 if (offset < opts.prefix) 4337 continue; 4338 4339 if (!elf_create_symbol(file->elf, name, func->sec, 4340 GELF_ST_BIND(func->sym.st_info), 4341 GELF_ST_TYPE(func->sym.st_info), 4342 prev->offset, opts.prefix)) 4343 return -1; 4344 4345 break; 4346 } 4347 4348 if (!prev) 4349 return 0; 4350 4351 if (!insn->cfi) { 4352 /* 4353 * This can happen if stack validation isn't enabled or the 4354 * function is annotated with STACK_FRAME_NON_STANDARD. 4355 */ 4356 return 0; 4357 } 4358 4359 /* Propagate insn->cfi to the prefix code */ 4360 cfi = cfi_hash_find_or_add(insn->cfi); 4361 for (; prev != insn; prev = next_insn_same_sec(file, prev)) 4362 prev->cfi = cfi; 4363 4364 return 0; 4365 } 4366 4367 static int create_prefix_symbols(struct objtool_file *file) 4368 { 4369 struct section *sec; 4370 struct symbol *func; 4371 4372 for_each_sec(file->elf, sec) { 4373 if (!is_text_sec(sec)) 4374 continue; 4375 4376 sec_for_each_sym(sec, func) { 4377 if (create_prefix_symbol(file, func)) 4378 return -1; 4379 } 4380 } 4381 4382 return 0; 4383 } 4384 4385 static int validate_symbol(struct objtool_file *file, struct section *sec, 4386 struct symbol *sym, struct insn_state *state) 4387 { 4388 struct instruction *insn; 4389 struct symbol *func; 4390 int ret; 4391 4392 if (!sym->len) { 4393 WARN("%s() is missing an ELF size annotation", sym->name); 4394 return 1; 4395 } 4396 4397 if (sym->pfunc != sym || is_alias_sym(sym)) 4398 return 0; 4399 4400 insn = find_insn(file, sec, sym->offset); 4401 if (!insn || insn->visited) 4402 return 0; 4403 4404 if (opts.uaccess) 4405 state->uaccess = sym->uaccess_safe; 4406 4407 func = insn_func(insn); 4408 4409 if (opts.trace && !fnmatch(opts.trace, sym->name, 0)) { 4410 trace_enable(); 4411 TRACE("%s: validation begin\n", sym->name); 4412 } 4413 4414 ret = validate_branch(file, func, insn, *state); 4415 if (ret) 4416 BT_INSN(insn, "<=== (sym)"); 4417 4418 TRACE("%s: validation %s\n\n", sym->name, ret ? "failed" : "end"); 4419 trace_disable(); 4420 4421 return ret; 4422 } 4423 4424 static int validate_section(struct objtool_file *file, struct section *sec) 4425 { 4426 struct insn_state state; 4427 struct symbol *func; 4428 int warnings = 0; 4429 4430 sec_for_each_sym(sec, func) { 4431 if (!is_func_sym(func)) 4432 continue; 4433 4434 init_insn_state(file, &state, sec); 4435 set_func_state(&state.cfi); 4436 4437 warnings += validate_symbol(file, sec, func, &state); 4438 } 4439 4440 return warnings; 4441 } 4442 4443 static int validate_noinstr_sections(struct objtool_file *file) 4444 { 4445 struct section *sec; 4446 int warnings = 0; 4447 4448 sec = find_section_by_name(file->elf, ".noinstr.text"); 4449 if (sec) { 4450 warnings += validate_section(file, sec); 4451 warnings += validate_unwind_hints(file, sec); 4452 } 4453 4454 sec = find_section_by_name(file->elf, ".entry.text"); 4455 if (sec) { 4456 warnings += validate_section(file, sec); 4457 warnings += validate_unwind_hints(file, sec); 4458 } 4459 4460 sec = find_section_by_name(file->elf, ".cpuidle.text"); 4461 if (sec) { 4462 warnings += validate_section(file, sec); 4463 warnings += validate_unwind_hints(file, sec); 4464 } 4465 4466 return warnings; 4467 } 4468 4469 static int validate_functions(struct objtool_file *file) 4470 { 4471 struct section *sec; 4472 int warnings = 0; 4473 4474 for_each_sec(file->elf, sec) { 4475 if (!is_text_sec(sec)) 4476 continue; 4477 4478 warnings += validate_section(file, sec); 4479 } 4480 4481 return warnings; 4482 } 4483 4484 static void mark_endbr_used(struct instruction *insn) 4485 { 4486 if (!list_empty(&insn->call_node)) 4487 list_del_init(&insn->call_node); 4488 } 4489 4490 static bool noendbr_range(struct objtool_file *file, struct instruction *insn) 4491 { 4492 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1); 4493 struct instruction *first; 4494 4495 if (!sym) 4496 return false; 4497 4498 first = find_insn(file, sym->sec, sym->offset); 4499 if (!first) 4500 return false; 4501 4502 if (first->type != INSN_ENDBR && !first->noendbr) 4503 return false; 4504 4505 return insn->offset == sym->offset + sym->len; 4506 } 4507 4508 static int __validate_ibt_insn(struct objtool_file *file, struct instruction *insn, 4509 struct instruction *dest) 4510 { 4511 if (dest->type == INSN_ENDBR) { 4512 mark_endbr_used(dest); 4513 return 0; 4514 } 4515 4516 if (insn_func(dest) && insn_func(insn) && 4517 insn_func(dest)->pfunc == insn_func(insn)->pfunc) { 4518 /* 4519 * Anything from->to self is either _THIS_IP_ or 4520 * IRET-to-self. 4521 * 4522 * There is no sane way to annotate _THIS_IP_ since the 4523 * compiler treats the relocation as a constant and is 4524 * happy to fold in offsets, skewing any annotation we 4525 * do, leading to vast amounts of false-positives. 4526 * 4527 * There's also compiler generated _THIS_IP_ through 4528 * KCOV and such which we have no hope of annotating. 4529 * 4530 * As such, blanket accept self-references without 4531 * issue. 4532 */ 4533 return 0; 4534 } 4535 4536 /* 4537 * Accept anything ANNOTATE_NOENDBR. 4538 */ 4539 if (dest->noendbr) 4540 return 0; 4541 4542 /* 4543 * Accept if this is the instruction after a symbol 4544 * that is (no)endbr -- typical code-range usage. 4545 */ 4546 if (noendbr_range(file, dest)) 4547 return 0; 4548 4549 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4550 return 1; 4551 } 4552 4553 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn) 4554 { 4555 struct instruction *dest; 4556 struct reloc *reloc; 4557 unsigned long off; 4558 int warnings = 0; 4559 4560 /* 4561 * Looking for function pointer load relocations. Ignore 4562 * direct/indirect branches: 4563 */ 4564 switch (insn->type) { 4565 4566 case INSN_CALL: 4567 case INSN_CALL_DYNAMIC: 4568 case INSN_JUMP_CONDITIONAL: 4569 case INSN_JUMP_UNCONDITIONAL: 4570 case INSN_JUMP_DYNAMIC: 4571 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4572 case INSN_RETURN: 4573 case INSN_NOP: 4574 return 0; 4575 4576 case INSN_LEA_RIP: 4577 if (!insn_reloc(file, insn)) { 4578 /* local function pointer reference without reloc */ 4579 4580 off = arch_jump_destination(insn); 4581 4582 dest = find_insn(file, insn->sec, off); 4583 if (!dest) { 4584 WARN_INSN(insn, "corrupt function pointer reference"); 4585 return 1; 4586 } 4587 4588 return __validate_ibt_insn(file, insn, dest); 4589 } 4590 break; 4591 4592 default: 4593 break; 4594 } 4595 4596 for (reloc = insn_reloc(file, insn); 4597 reloc; 4598 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 4599 reloc_offset(reloc) + 1, 4600 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) { 4601 4602 off = reloc->sym->offset + arch_insn_adjusted_addend(insn, reloc); 4603 4604 dest = find_insn(file, reloc->sym->sec, off); 4605 if (!dest) 4606 continue; 4607 4608 warnings += __validate_ibt_insn(file, insn, dest); 4609 } 4610 4611 return warnings; 4612 } 4613 4614 static int validate_ibt_data_reloc(struct objtool_file *file, 4615 struct reloc *reloc) 4616 { 4617 struct instruction *dest; 4618 4619 dest = find_insn(file, reloc->sym->sec, 4620 reloc->sym->offset + reloc_addend(reloc)); 4621 if (!dest) 4622 return 0; 4623 4624 if (dest->type == INSN_ENDBR) { 4625 mark_endbr_used(dest); 4626 return 0; 4627 } 4628 4629 if (dest->noendbr) 4630 return 0; 4631 4632 WARN_FUNC(reloc->sec->base, reloc_offset(reloc), 4633 "data relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4634 4635 return 1; 4636 } 4637 4638 /* 4639 * Validate IBT rules and remove used ENDBR instructions from the seal list. 4640 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with 4641 * NOPs) later, in create_ibt_endbr_seal_sections(). 4642 */ 4643 static int validate_ibt(struct objtool_file *file) 4644 { 4645 struct section *sec; 4646 struct reloc *reloc; 4647 struct instruction *insn; 4648 int warnings = 0; 4649 4650 for_each_insn(file, insn) 4651 warnings += validate_ibt_insn(file, insn); 4652 4653 for_each_sec(file->elf, sec) { 4654 4655 /* Already done by validate_ibt_insn() */ 4656 if (is_text_sec(sec)) 4657 continue; 4658 4659 if (!sec->rsec) 4660 continue; 4661 4662 /* 4663 * These sections can reference text addresses, but not with 4664 * the intent to indirect branch to them. 4665 */ 4666 if ((!strncmp(sec->name, ".discard", 8) && 4667 strcmp(sec->name, ".discard.ibt_endbr_noseal")) || 4668 !strncmp(sec->name, ".debug", 6) || 4669 !strcmp(sec->name, ".altinstructions") || 4670 !strcmp(sec->name, ".ibt_endbr_seal") || 4671 !strcmp(sec->name, ".kcfi_traps") || 4672 !strcmp(sec->name, ".orc_unwind_ip") || 4673 !strcmp(sec->name, ".retpoline_sites") || 4674 !strcmp(sec->name, ".smp_locks") || 4675 !strcmp(sec->name, ".static_call_sites") || 4676 !strcmp(sec->name, "_error_injection_whitelist") || 4677 !strcmp(sec->name, "_kprobe_blacklist") || 4678 !strcmp(sec->name, "__bug_table") || 4679 !strcmp(sec->name, "__ex_table") || 4680 !strcmp(sec->name, "__jump_table") || 4681 !strcmp(sec->name, ".init.klp_funcs") || 4682 !strcmp(sec->name, "__mcount_loc") || 4683 !strcmp(sec->name, ".llvm.call-graph-profile") || 4684 !strcmp(sec->name, ".llvm_bb_addr_map") || 4685 !strcmp(sec->name, "__tracepoints") || 4686 !strcmp(sec->name, ".return_sites") || 4687 !strcmp(sec->name, ".call_sites") || 4688 !strcmp(sec->name, "__patchable_function_entries")) 4689 continue; 4690 4691 for_each_reloc(sec->rsec, reloc) 4692 warnings += validate_ibt_data_reloc(file, reloc); 4693 } 4694 4695 return warnings; 4696 } 4697 4698 static int validate_sls(struct objtool_file *file) 4699 { 4700 struct instruction *insn, *next_insn; 4701 int warnings = 0; 4702 4703 for_each_insn(file, insn) { 4704 next_insn = next_insn_same_sec(file, insn); 4705 4706 if (insn->retpoline_safe) 4707 continue; 4708 4709 switch (insn->type) { 4710 case INSN_RETURN: 4711 if (!next_insn || next_insn->type != INSN_TRAP) { 4712 WARN_INSN(insn, "missing int3 after ret"); 4713 warnings++; 4714 } 4715 4716 break; 4717 case INSN_JUMP_DYNAMIC: 4718 if (!next_insn || next_insn->type != INSN_TRAP) { 4719 WARN_INSN(insn, "missing int3 after indirect jump"); 4720 warnings++; 4721 } 4722 break; 4723 default: 4724 break; 4725 } 4726 } 4727 4728 return warnings; 4729 } 4730 4731 static int validate_reachable_instructions(struct objtool_file *file) 4732 { 4733 struct instruction *insn, *prev_insn; 4734 struct symbol *call_dest; 4735 int warnings = 0; 4736 4737 if (file->ignore_unreachables) 4738 return 0; 4739 4740 for_each_insn(file, insn) { 4741 if (insn->visited || ignore_unreachable_insn(file, insn)) 4742 continue; 4743 4744 prev_insn = prev_insn_same_sec(file, insn); 4745 if (prev_insn && prev_insn->dead_end) { 4746 call_dest = insn_call_dest(prev_insn); 4747 if (call_dest) { 4748 WARN_INSN(insn, "%s() missing __noreturn in .c/.h or NORETURN() in noreturns.h", 4749 call_dest->name); 4750 warnings++; 4751 continue; 4752 } 4753 } 4754 4755 WARN_INSN(insn, "unreachable instruction"); 4756 warnings++; 4757 } 4758 4759 return warnings; 4760 } 4761 4762 __weak bool arch_absolute_reloc(struct elf *elf, struct reloc *reloc) 4763 { 4764 unsigned int type = reloc_type(reloc); 4765 size_t sz = elf_addr_size(elf); 4766 4767 return (sz == 8) ? (type == R_ABS64) : (type == R_ABS32); 4768 } 4769 4770 static int check_abs_references(struct objtool_file *file) 4771 { 4772 struct section *sec; 4773 struct reloc *reloc; 4774 int ret = 0; 4775 4776 for_each_sec(file->elf, sec) { 4777 /* absolute references in non-loadable sections are fine */ 4778 if (!(sec->sh.sh_flags & SHF_ALLOC)) 4779 continue; 4780 4781 /* section must have an associated .rela section */ 4782 if (!sec->rsec) 4783 continue; 4784 4785 /* 4786 * Special case for compiler generated metadata that is not 4787 * consumed until after boot. 4788 */ 4789 if (!strcmp(sec->name, "__patchable_function_entries")) 4790 continue; 4791 4792 for_each_reloc(sec->rsec, reloc) { 4793 if (arch_absolute_reloc(file->elf, reloc)) { 4794 WARN("section %s has absolute relocation at offset 0x%llx", 4795 sec->name, (unsigned long long)reloc_offset(reloc)); 4796 ret++; 4797 } 4798 } 4799 } 4800 return ret; 4801 } 4802 4803 struct insn_chunk { 4804 void *addr; 4805 struct insn_chunk *next; 4806 }; 4807 4808 /* 4809 * Reduce peak RSS usage by freeing insns memory before writing the ELF file, 4810 * which can trigger more allocations for .debug_* sections whose data hasn't 4811 * been read yet. 4812 */ 4813 void free_insns(struct objtool_file *file) 4814 { 4815 struct instruction *insn; 4816 struct insn_chunk *chunks = NULL, *chunk; 4817 4818 for_each_insn(file, insn) { 4819 if (!insn->idx) { 4820 chunk = malloc(sizeof(*chunk)); 4821 chunk->addr = insn; 4822 chunk->next = chunks; 4823 chunks = chunk; 4824 } 4825 } 4826 4827 for (chunk = chunks; chunk; chunk = chunk->next) 4828 free(chunk->addr); 4829 } 4830 4831 const char *objtool_disas_insn(struct instruction *insn) 4832 { 4833 struct disas_context *dctx = objtool_disas_ctx; 4834 4835 if (!dctx) 4836 return ""; 4837 4838 disas_insn(dctx, insn); 4839 return disas_result(dctx); 4840 } 4841 4842 int check(struct objtool_file *file) 4843 { 4844 struct disas_context *disas_ctx = NULL; 4845 int ret = 0, warnings = 0; 4846 4847 /* 4848 * Create a disassembly context if we might disassemble any 4849 * instruction or function. 4850 */ 4851 if (opts.verbose || opts.backtrace || opts.trace || opts.disas) { 4852 disas_ctx = disas_context_create(file); 4853 if (!disas_ctx) { 4854 opts.disas = false; 4855 opts.trace = false; 4856 } 4857 objtool_disas_ctx = disas_ctx; 4858 } 4859 4860 ret = decode_file(file); 4861 if (ret) 4862 goto out; 4863 4864 if (!nr_insns) 4865 goto out; 4866 4867 if (opts.retpoline) 4868 warnings += validate_retpoline(file); 4869 4870 if (validate_branch_enabled()) { 4871 int w = 0; 4872 4873 w += validate_functions(file); 4874 w += validate_unwind_hints(file, NULL); 4875 if (!w) 4876 w += validate_reachable_instructions(file); 4877 4878 warnings += w; 4879 4880 } else if (opts.noinstr) { 4881 warnings += validate_noinstr_sections(file); 4882 } 4883 4884 if (opts.unret) { 4885 /* 4886 * Must be after validate_branch() and friends, it plays 4887 * further games with insn->visited. 4888 */ 4889 warnings += validate_unrets(file); 4890 } 4891 4892 if (opts.ibt) 4893 warnings += validate_ibt(file); 4894 4895 if (opts.sls) 4896 warnings += validate_sls(file); 4897 4898 if (opts.static_call) { 4899 ret = create_static_call_sections(file); 4900 if (ret) 4901 goto out; 4902 } 4903 4904 if (opts.retpoline) { 4905 ret = create_retpoline_sites_sections(file); 4906 if (ret) 4907 goto out; 4908 } 4909 4910 if (opts.cfi) { 4911 ret = create_cfi_sections(file); 4912 if (ret) 4913 goto out; 4914 } 4915 4916 if (opts.rethunk) { 4917 ret = create_return_sites_sections(file); 4918 if (ret) 4919 goto out; 4920 4921 if (opts.hack_skylake) { 4922 ret = create_direct_call_sections(file); 4923 if (ret) 4924 goto out; 4925 } 4926 } 4927 4928 if (opts.mcount) { 4929 ret = create_mcount_loc_sections(file); 4930 if (ret) 4931 goto out; 4932 } 4933 4934 if (opts.prefix) { 4935 ret = create_prefix_symbols(file); 4936 if (ret) 4937 goto out; 4938 } 4939 4940 if (opts.ibt) { 4941 ret = create_ibt_endbr_seal_sections(file); 4942 if (ret) 4943 goto out; 4944 } 4945 4946 if (opts.noabs) 4947 warnings += check_abs_references(file); 4948 4949 if (opts.checksum) { 4950 ret = calculate_checksums(file); 4951 if (ret) 4952 goto out; 4953 ret = create_sym_checksum_section(file); 4954 if (ret) 4955 goto out; 4956 } 4957 4958 if (opts.orc && nr_insns) { 4959 ret = orc_create(file); 4960 if (ret) 4961 goto out; 4962 } 4963 4964 if (opts.stats) { 4965 printf("nr_insns_visited: %ld\n", nr_insns_visited); 4966 printf("nr_cfi: %ld\n", nr_cfi); 4967 printf("nr_cfi_reused: %ld\n", nr_cfi_reused); 4968 printf("nr_cfi_cache: %ld\n", nr_cfi_cache); 4969 } 4970 4971 out: 4972 if (ret || warnings) { 4973 if (opts.werror && warnings) 4974 ret = 1; 4975 4976 if (opts.verbose) { 4977 if (opts.werror && warnings) 4978 WARN("%d warning(s) upgraded to errors", warnings); 4979 disas_warned_funcs(disas_ctx); 4980 } 4981 } 4982 4983 if (opts.disas) 4984 disas_funcs(disas_ctx); 4985 4986 if (disas_ctx) { 4987 disas_context_destroy(disas_ctx); 4988 objtool_disas_ctx = NULL; 4989 } 4990 4991 free_insns(file); 4992 4993 if (!ret && !warnings) 4994 return 0; 4995 4996 if (opts.backup && make_backup()) 4997 return 1; 4998 4999 return ret; 5000 } 5001