1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #define _GNU_SOURCE /* memmem() */ 3 #include <subcmd/parse-options.h> 4 #include <stdlib.h> 5 #include <string.h> 6 #include <libgen.h> 7 #include <stdio.h> 8 #include <ctype.h> 9 10 #include <objtool/objtool.h> 11 #include <objtool/warn.h> 12 #include <objtool/arch.h> 13 #include <objtool/klp.h> 14 #include <objtool/util.h> 15 #include <arch/special.h> 16 17 #include <linux/align.h> 18 #include <linux/objtool_types.h> 19 #include <linux/livepatch_external.h> 20 #include <linux/stringify.h> 21 #include <linux/string.h> 22 #include <linux/jhash.h> 23 24 #define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) 25 26 struct elfs { 27 struct elf *orig, *patched, *out; 28 const char *modname; 29 }; 30 31 struct export { 32 struct hlist_node hash; 33 char *mod, *sym; 34 }; 35 36 static const char * const klp_diff_usage[] = { 37 "objtool klp diff [<options>] <in1.o> <in2.o> <out.o>", 38 NULL, 39 }; 40 41 static const struct option klp_diff_options[] = { 42 OPT_GROUP("Options:"), 43 OPT_BOOLEAN('d', "debug", &debug, "enable debug output"), 44 OPT_END(), 45 }; 46 47 static DEFINE_HASHTABLE(exports, 15); 48 49 static char *escape_str(const char *orig) 50 { 51 size_t len = 0; 52 const char *a; 53 char *b, *new; 54 55 for (a = orig; *a; a++) { 56 switch (*a) { 57 case '\001': len += 5; break; 58 case '\n': 59 case '\t': len += 2; break; 60 default: len++; 61 } 62 } 63 64 new = malloc(len + 1); 65 if (!new) 66 return NULL; 67 68 for (a = orig, b = new; *a; a++) { 69 switch (*a) { 70 case '\001': memcpy(b, "<SOH>", 5); b += 5; break; 71 case '\n': *b++ = '\\'; *b++ = 'n'; break; 72 case '\t': *b++ = '\\'; *b++ = 't'; break; 73 default: *b++ = *a; 74 } 75 } 76 77 *b = '\0'; 78 return new; 79 } 80 81 static int read_exports(void) 82 { 83 const char *symvers = "Module.symvers"; 84 char line[1024], *path = NULL; 85 unsigned int line_num = 1; 86 FILE *file; 87 88 file = fopen(symvers, "r"); 89 if (!file) { 90 path = top_level_dir(symvers); 91 if (!path) { 92 ERROR("can't open '%s', \"objtool diff\" should be run from the kernel tree", symvers); 93 return -1; 94 } 95 96 file = fopen(path, "r"); 97 if (!file) { 98 ERROR_GLIBC("fopen"); 99 return -1; 100 } 101 } 102 103 while (fgets(line, 1024, file)) { 104 char *sym, *mod, *type; 105 struct export *export; 106 107 sym = strchr(line, '\t'); 108 if (!sym) { 109 ERROR("malformed Module.symvers (sym) at line %d", line_num); 110 return -1; 111 } 112 113 *sym++ = '\0'; 114 115 mod = strchr(sym, '\t'); 116 if (!mod) { 117 ERROR("malformed Module.symvers (mod) at line %d", line_num); 118 return -1; 119 } 120 121 *mod++ = '\0'; 122 123 type = strchr(mod, '\t'); 124 if (!type) { 125 ERROR("malformed Module.symvers (type) at line %d", line_num); 126 return -1; 127 } 128 129 *type++ = '\0'; 130 131 if (*sym == '\0' || *mod == '\0') { 132 ERROR("malformed Module.symvers at line %d", line_num); 133 return -1; 134 } 135 136 export = calloc(1, sizeof(*export)); 137 if (!export) { 138 ERROR_GLIBC("calloc"); 139 return -1; 140 } 141 142 export->mod = strdup(mod); 143 if (!export->mod) { 144 ERROR_GLIBC("strdup"); 145 return -1; 146 } 147 148 export->sym = strdup(sym); 149 if (!export->sym) { 150 ERROR_GLIBC("strdup"); 151 return -1; 152 } 153 154 hash_add(exports, &export->hash, str_hash(sym)); 155 } 156 157 free(path); 158 fclose(file); 159 160 return 0; 161 } 162 163 static int read_sym_checksums(struct elf *elf) 164 { 165 struct section *sec; 166 167 sec = find_section_by_name(elf, ".discard.sym_checksum"); 168 if (!sec) { 169 ERROR("'%s' missing .discard.sym_checksum section, file not processed by 'objtool --checksum'?", 170 elf->name); 171 return -1; 172 } 173 174 if (!sec->rsec) { 175 ERROR("missing reloc section for .discard.sym_checksum"); 176 return -1; 177 } 178 179 if (sec_size(sec) % sizeof(struct sym_checksum)) { 180 ERROR("struct sym_checksum size mismatch"); 181 return -1; 182 } 183 184 for (int i = 0; i < sec_size(sec) / sizeof(struct sym_checksum); i++) { 185 struct sym_checksum *sym_checksum; 186 struct reloc *reloc; 187 struct symbol *sym; 188 189 sym_checksum = (struct sym_checksum *)sec->data->d_buf + i; 190 191 reloc = find_reloc_by_dest(elf, sec, i * sizeof(*sym_checksum)); 192 if (!reloc) { 193 ERROR("can't find reloc for sym_checksum[%d]", i); 194 return -1; 195 } 196 197 sym = reloc->sym; 198 199 if (is_sec_sym(sym)) { 200 ERROR("not sure how to handle section %s", sym->name); 201 return -1; 202 } 203 204 if (is_func_sym(sym)) 205 sym->csum.checksum = sym_checksum->checksum; 206 } 207 208 return 0; 209 } 210 211 static struct symbol *first_file_symbol(struct elf *elf) 212 { 213 struct symbol *sym; 214 215 for_each_sym(elf, sym) { 216 if (is_file_sym(sym)) 217 return sym; 218 } 219 220 return NULL; 221 } 222 223 static struct symbol *next_file_symbol(struct elf *elf, struct symbol *sym) 224 { 225 for_each_sym_continue(elf, sym) { 226 if (is_file_sym(sym)) 227 return sym; 228 } 229 230 return NULL; 231 } 232 233 /* 234 * Certain static local variables should never be correlated. They will be 235 * used in place rather than referencing the originals. 236 */ 237 static bool is_uncorrelated_static_local(struct symbol *sym) 238 { 239 static const char * const vars[] = { 240 "__already_done", 241 "__func__", 242 "__key", 243 "__warned", 244 "_entry", 245 "_entry_ptr", 246 "_rs", 247 "descriptor", 248 "CSWTCH", 249 }; 250 const char *dot; 251 252 if (!is_object_sym(sym) || !is_local_sym(sym)) 253 return false; 254 255 /* WARN_ONCE, etc */ 256 if (!strcmp(sym->sec->name, ".data..once")) 257 return true; 258 259 dot = strchr(sym->name, '.'); 260 if (!dot) 261 return false; 262 263 for (int i = 0; i < ARRAY_SIZE(vars); i++) { 264 size_t len = strlen(vars[i]); 265 266 /* GCC: <var>.<id> */ 267 if (strstarts(sym->name, vars[i]) && (sym->name[len] == '.')) 268 return true; 269 270 /* Clang: <func>.<var>[.<id>] */ 271 if (strstarts(dot + 1, vars[i]) && 272 (dot[1 + len] == '.' || dot[1 + len] == '\0')) 273 return true; 274 } 275 276 return false; 277 } 278 279 /* 280 * .L symbols are assembler-local labels not present in kallsyms. They must 281 * never become KLP relocations; instead their data is cloned into the patch 282 * module. This covers .Ltmp* (Clang temp labels), .L__const.* (Clang local 283 * constants), and any other assembler-local pattern. 284 */ 285 static bool is_local_label(struct symbol *sym) 286 { 287 return strstarts(sym->name, ".L"); 288 } 289 290 static bool is_special_section(struct section *sec) 291 { 292 static const char * const specials[] = { 293 ".altinstructions", 294 ".kcfi_traps", 295 ".smp_locks", 296 "__bug_table", 297 "__ex_table", 298 "__jump_table", 299 "__mcount_loc", 300 301 /* 302 * Extract .static_call_sites here to inherit non-module 303 * preferential treatment. The later static call processing 304 * during klp module build will be skipped when it sees this 305 * section already exists. 306 */ 307 ".static_call_sites", 308 }; 309 310 static const char * const non_special_discards[] = { 311 ".discard.addressable", 312 ".discard.sym_checksum", 313 }; 314 315 if (is_text_sec(sec)) 316 return false; 317 318 for (int i = 0; i < ARRAY_SIZE(specials); i++) { 319 if (!strcmp(sec->name, specials[i])) 320 return true; 321 } 322 323 /* Most .discard data sections are special */ 324 for (int i = 0; i < ARRAY_SIZE(non_special_discards); i++) { 325 if (!strcmp(sec->name, non_special_discards[i])) 326 return false; 327 } 328 329 return strstarts(sec->name, ".discard."); 330 } 331 332 /* 333 * These sections are referenced by special sections but aren't considered 334 * special sections themselves. 335 */ 336 static bool is_special_section_aux(struct section *sec) 337 { 338 static const char * const specials_aux[] = { 339 ".altinstr_replacement", 340 ".altinstr_aux", 341 }; 342 343 for (int i = 0; i < ARRAY_SIZE(specials_aux); i++) { 344 if (!strcmp(sec->name, specials_aux[i])) 345 return true; 346 } 347 348 return false; 349 } 350 351 /* 352 * Symbols created by ___ADDRESSABLE() are only used to convince the toolchain 353 * not to optimize out the referenced symbol. 354 */ 355 static bool is_addressable_sym(struct symbol *sym) 356 { 357 return !strcmp(sym->sec->name, ".discard.addressable"); 358 } 359 360 /* 361 * ABS symbols are typically assembly .set/.equ constants which are never 362 * referenced by relocations. (Exclude FILE symbols which are also SHN_ABS.) 363 */ 364 static bool is_abs_sym(struct symbol *sym) 365 { 366 return sym->sym.st_shndx == SHN_ABS && !is_file_sym(sym); 367 } 368 369 static bool is_initcall_sym(struct symbol *sym) 370 { 371 return strstarts(sym->name, "__initcall__") || 372 strstarts(sym->name, "__initstub__"); 373 } 374 375 /* 376 * Some .rodata is anonymous and can't be correlated due to there being no 377 * symbol names. 378 * 379 * The .rodata.cst* sections aren't technically anonymous, they're SHF_MERGE 380 * constant pool sections containing small fixed-size data (lookup tables, 381 * bitmasks) which are only read by value, so pointer equivalence isn't needed. 382 * They are typically referenced by UBSAN data sections. 383 */ 384 static bool is_anonymous_rodata(struct symbol *sym) 385 { 386 return is_rodata_sec(sym->sec) && 387 (!is_object_sym(sym) || strstarts(sym->sec->name, ".rodata.cst")); 388 } 389 390 /* 391 * These symbols should never be correlated, so their local patched versions 392 * are used instead of linking to the originals. 393 */ 394 static bool dont_correlate(struct symbol *sym) 395 { 396 return is_file_sym(sym) || 397 is_null_sym(sym) || 398 is_sec_sym(sym) || 399 is_abs_sym(sym) || 400 is_prefix_func(sym) || 401 is_uncorrelated_static_local(sym) || 402 is_local_label(sym) || 403 is_string_sec(sym->sec) || 404 is_anonymous_rodata(sym) || 405 is_initcall_sym(sym) || 406 is_addressable_sym(sym) || 407 is_special_section(sym->sec) || 408 is_special_section_aux(sym->sec); 409 } 410 411 /* 412 * When there is no full name match, try match demangled_name. This would 413 * match original foo.llvm.123 to patched foo.llvm.456. 414 * 415 * Note that, in very rare cases, it is possible to have multiple 416 * foo.llvm.<hash> in the same kernel. When this happens, report error and 417 * fail the diff. 418 */ 419 static int find_global_symbol_by_demangled_name(struct elf *elf, struct symbol *sym, 420 struct symbol **out_sym) 421 { 422 struct symbol *sym2, *result = NULL; 423 int count = 0; 424 425 for_each_sym_by_demangled_name(elf, sym->demangled_name, sym2) { 426 if (is_local_sym(sym2) || sym2->twin) 427 continue; 428 429 count++; 430 result = sym2; 431 } 432 433 if (count > 1) { 434 ERROR("Multiple (%d) correlation candidates for %s", count, sym->name); 435 return -1; 436 } 437 438 *out_sym = result; 439 return 0; 440 } 441 442 /* 443 * For each symbol in the original kernel, find its corresponding "twin" in the 444 * patched kernel. 445 */ 446 static int correlate_symbols(struct elfs *e) 447 { 448 struct symbol *file1_sym, *file2_sym; 449 struct symbol *sym1, *sym2; 450 451 file1_sym = first_file_symbol(e->orig); 452 file2_sym = first_file_symbol(e->patched); 453 454 /* 455 * Correlate any locals before the first FILE symbol. This has been 456 * seen when LTO inexplicably strips the initramfs_data.o FILE symbol 457 * due to the file only containing data and no code. 458 */ 459 for_each_sym(e->orig, sym1) { 460 if (sym1 == file1_sym || !is_local_sym(sym1)) 461 break; 462 463 if (dont_correlate(sym1)) 464 continue; 465 466 for_each_sym(e->patched, sym2) { 467 if (sym2 == file2_sym || !is_local_sym(sym2)) 468 break; 469 470 if (sym2->twin || dont_correlate(sym2)) 471 continue; 472 473 if (strcmp(sym1->demangled_name, sym2->demangled_name)) 474 continue; 475 476 sym1->twin = sym2; 477 sym2->twin = sym1; 478 break; 479 } 480 } 481 482 /* Correlate locals after the first FILE symbol */ 483 for (; ; file1_sym = next_file_symbol(e->orig, file1_sym), 484 file2_sym = next_file_symbol(e->patched, file2_sym)) { 485 486 if (!file1_sym && file2_sym) { 487 ERROR("FILE symbol mismatch: NULL != %s", file2_sym->name); 488 return -1; 489 } 490 491 if (file1_sym && !file2_sym) { 492 ERROR("FILE symbol mismatch: %s != NULL", file1_sym->name); 493 return -1; 494 } 495 496 if (!file1_sym) 497 break; 498 499 if (strcmp(file1_sym->name, file2_sym->name)) { 500 ERROR("FILE symbol mismatch: %s != %s", file1_sym->name, file2_sym->name); 501 return -1; 502 } 503 504 file1_sym->twin = file2_sym; 505 file2_sym->twin = file1_sym; 506 507 sym1 = file1_sym; 508 509 for_each_sym_continue(e->orig, sym1) { 510 if (is_file_sym(sym1) || !is_local_sym(sym1)) 511 break; 512 513 if (dont_correlate(sym1)) 514 continue; 515 516 sym2 = file2_sym; 517 for_each_sym_continue(e->patched, sym2) { 518 if (is_file_sym(sym2) || !is_local_sym(sym2)) 519 break; 520 521 if (sym2->twin || dont_correlate(sym2)) 522 continue; 523 524 if (strcmp(sym1->demangled_name, sym2->demangled_name)) 525 continue; 526 527 sym1->twin = sym2; 528 sym2->twin = sym1; 529 break; 530 } 531 } 532 } 533 534 /* Correlate globals */ 535 for_each_sym(e->orig, sym1) { 536 if (sym1->bind == STB_LOCAL) 537 continue; 538 539 sym2 = find_global_symbol_by_name(e->patched, sym1->name); 540 if (sym2 && !sym2->twin) { 541 sym1->twin = sym2; 542 sym2->twin = sym1; 543 } 544 } 545 546 /* 547 * Correlate globals with demangled_name. 548 * A separate loop is needed because we want to finish all the 549 * full name correlations first. 550 */ 551 for_each_sym(e->orig, sym1) { 552 if (sym1->bind == STB_LOCAL || sym1->twin) 553 continue; 554 555 if (find_global_symbol_by_demangled_name(e->patched, sym1, &sym2)) 556 return -1; 557 558 if (sym2 && !sym2->twin) { 559 sym1->twin = sym2; 560 sym2->twin = sym1; 561 } 562 } 563 564 /* Correlate original locals with patched globals */ 565 for_each_sym(e->orig, sym1) { 566 if (sym1->twin || dont_correlate(sym1) || !is_local_sym(sym1)) 567 continue; 568 569 sym2 = find_global_symbol_by_name(e->patched, sym1->name); 570 if (!sym2 && find_global_symbol_by_demangled_name(e->patched, sym1, &sym2)) 571 return -1; 572 573 if (sym2 && !sym2->twin) { 574 sym1->twin = sym2; 575 sym2->twin = sym1; 576 } 577 } 578 579 /* Correlate original globals with patched locals */ 580 for_each_sym(e->patched, sym2) { 581 if (sym2->twin || dont_correlate(sym2) || !is_local_sym(sym2)) 582 continue; 583 584 sym1 = find_global_symbol_by_name(e->orig, sym2->name); 585 if (!sym1 && find_global_symbol_by_demangled_name(e->orig, sym2, &sym1)) 586 return -1; 587 588 if (sym1 && !sym1->twin) { 589 sym2->twin = sym1; 590 sym1->twin = sym2; 591 } 592 } 593 594 for_each_sym(e->orig, sym1) { 595 if (sym1->twin || dont_correlate(sym1)) 596 continue; 597 WARN("no correlation: %s", sym1->name); 598 } 599 600 return 0; 601 } 602 603 /* "sympos" is used by livepatch to disambiguate duplicate symbol names */ 604 static unsigned long find_sympos(struct elf *elf, struct symbol *sym) 605 { 606 bool vmlinux = str_ends_with(objname, "vmlinux.o"); 607 unsigned long sympos = 0, nr_matches = 0; 608 bool has_dup = false; 609 struct symbol *s; 610 611 if (sym->bind != STB_LOCAL) 612 return 0; 613 614 if (vmlinux && is_func_sym(sym)) { 615 /* 616 * HACK: Unfortunately, symbol ordering can differ between 617 * vmlinux.o and vmlinux due to the linker script emitting 618 * .text.unlikely* before .text*. Count .text.unlikely* first. 619 * 620 * TODO: Disambiguate symbols more reliably (checksums?) 621 */ 622 for_each_sym(elf, s) { 623 if (strstarts(s->sec->name, ".text.unlikely") && 624 !strcmp(s->name, sym->name)) { 625 nr_matches++; 626 if (s == sym) 627 sympos = nr_matches; 628 else 629 has_dup = true; 630 } 631 } 632 for_each_sym(elf, s) { 633 if (!strstarts(s->sec->name, ".text.unlikely") && 634 !strcmp(s->name, sym->name)) { 635 nr_matches++; 636 if (s == sym) 637 sympos = nr_matches; 638 else 639 has_dup = true; 640 } 641 } 642 } else { 643 for_each_sym(elf, s) { 644 if (!strcmp(s->name, sym->name)) { 645 nr_matches++; 646 if (s == sym) 647 sympos = nr_matches; 648 else 649 has_dup = true; 650 } 651 } 652 } 653 654 if (!sympos) { 655 ERROR("can't find sympos for %s", sym->name); 656 return ULONG_MAX; 657 } 658 659 return has_dup ? sympos : 0; 660 } 661 662 static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym); 663 664 static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym, 665 bool data_too) 666 { 667 struct section *out_sec = NULL; 668 unsigned long offset = 0; 669 struct symbol *out_sym; 670 671 if (data_too && !is_undef_sym(patched_sym)) { 672 struct section *patched_sec = patched_sym->sec; 673 674 out_sec = find_section_by_name(elf, patched_sec->name); 675 if (!out_sec) { 676 out_sec = elf_create_section(elf, patched_sec->name, 0, 677 patched_sec->sh.sh_entsize, 678 patched_sec->sh.sh_type, 679 patched_sec->sh.sh_addralign, 680 patched_sec->sh.sh_flags); 681 if (!out_sec) 682 return NULL; 683 } 684 685 if (is_string_sec(patched_sym->sec)) { 686 out_sym = elf_create_section_symbol(elf, out_sec); 687 if (!out_sym) 688 return NULL; 689 690 goto sym_created; 691 } 692 693 if (!is_sec_sym(patched_sym)) 694 offset = ALIGN(sec_size(out_sec), out_sec->sh.sh_addralign); 695 696 if (patched_sym->len || is_sec_sym(patched_sym)) { 697 void *data = NULL; 698 size_t size; 699 700 /* bss doesn't have data */ 701 if (patched_sym->sec->data && patched_sym->sec->data->d_buf) 702 data = patched_sym->sec->data->d_buf + patched_sym->offset; 703 704 if (is_sec_sym(patched_sym)) 705 size = sec_size(patched_sym->sec); 706 else 707 size = patched_sym->len; 708 709 if (!elf_add_data(elf, out_sec, data, size)) 710 return NULL; 711 } 712 } 713 714 out_sym = elf_create_symbol(elf, patched_sym->name, out_sec, 715 patched_sym->bind, patched_sym->type, 716 offset, patched_sym->len); 717 if (!out_sym) 718 return NULL; 719 720 sym_created: 721 patched_sym->clone = out_sym; 722 out_sym->clone = patched_sym; 723 724 return out_sym; 725 } 726 727 static const char *sym_type(struct symbol *sym) 728 { 729 switch (sym->type) { 730 case STT_NOTYPE: return "NOTYPE"; 731 case STT_OBJECT: return "OBJECT"; 732 case STT_FUNC: return "FUNC"; 733 case STT_SECTION: return "SECTION"; 734 case STT_FILE: return "FILE"; 735 default: return "UNKNOWN"; 736 } 737 } 738 739 static const char *sym_bind(struct symbol *sym) 740 { 741 switch (sym->bind) { 742 case STB_LOCAL: return "LOCAL"; 743 case STB_GLOBAL: return "GLOBAL"; 744 case STB_WEAK: return "WEAK"; 745 default: return "UNKNOWN"; 746 } 747 } 748 749 /* 750 * Copy a symbol to the output object, optionally including its data and 751 * relocations. 752 */ 753 static struct symbol *clone_symbol(struct elfs *e, struct symbol *patched_sym, 754 bool data_too) 755 { 756 struct symbol *pfx; 757 758 if (patched_sym->clone) 759 return patched_sym->clone; 760 761 dbg_indent("%s%s", patched_sym->name, data_too ? " [+DATA]" : ""); 762 763 /* Make sure the prefix gets cloned first */ 764 if (is_func_sym(patched_sym) && data_too) { 765 pfx = get_func_prefix(patched_sym); 766 if (pfx) 767 clone_symbol(e, pfx, true); 768 } 769 770 if (!__clone_symbol(e->out, patched_sym, data_too)) 771 return NULL; 772 773 if (data_too && clone_sym_relocs(e, patched_sym)) 774 return NULL; 775 776 return patched_sym->clone; 777 } 778 779 static void mark_included_function(struct symbol *func) 780 { 781 struct symbol *pfx; 782 783 func->included = 1; 784 785 /* Include prefix function */ 786 pfx = get_func_prefix(func); 787 if (pfx) 788 pfx->included = 1; 789 790 /* Make sure .cold parent+child always stay together */ 791 if (func->cfunc && func->cfunc != func) 792 func->cfunc->included = 1; 793 if (func->pfunc && func->pfunc != func) 794 func->pfunc->included = 1; 795 } 796 797 /* 798 * Copy all changed functions (and their dependencies) from the patched object 799 * to the output object. 800 */ 801 static int mark_changed_functions(struct elfs *e) 802 { 803 struct symbol *sym_orig, *patched_sym; 804 bool changed = false; 805 806 /* Find changed functions */ 807 for_each_sym(e->orig, sym_orig) { 808 if (!is_func_sym(sym_orig) || dont_correlate(sym_orig)) 809 continue; 810 811 patched_sym = sym_orig->twin; 812 if (!patched_sym) 813 continue; 814 815 if (sym_orig->csum.checksum != patched_sym->csum.checksum) { 816 patched_sym->changed = 1; 817 mark_included_function(patched_sym); 818 changed = true; 819 } 820 } 821 822 /* Find added functions and print them */ 823 for_each_sym(e->patched, patched_sym) { 824 if (!is_func_sym(patched_sym) || dont_correlate(patched_sym)) 825 continue; 826 827 if (!patched_sym->twin) { 828 printf("%s: new function: %s\n", objname, patched_sym->name); 829 mark_included_function(patched_sym); 830 changed = true; 831 } 832 } 833 834 /* Print changed functions */ 835 for_each_sym(e->patched, patched_sym) { 836 if (patched_sym->changed) 837 printf("%s: changed function: %s\n", objname, patched_sym->name); 838 } 839 840 return !changed ? -1 : 0; 841 } 842 843 static int clone_included_functions(struct elfs *e) 844 { 845 struct symbol *patched_sym; 846 847 for_each_sym(e->patched, patched_sym) { 848 if (patched_sym->included) { 849 if (!clone_symbol(e, patched_sym, true)) 850 return -1; 851 } 852 } 853 854 return 0; 855 } 856 857 static struct export *find_export(struct symbol *sym) 858 { 859 struct export *export; 860 861 hash_for_each_possible(exports, export, hash, str_hash(sym->name)) { 862 if (!strcmp(export->sym, sym->name)) 863 return export; 864 } 865 866 return NULL; 867 } 868 869 static const char *__find_modname(struct elfs *e) 870 { 871 struct section *sec; 872 char *name; 873 874 sec = find_section_by_name(e->orig, ".modinfo"); 875 if (!sec) { 876 ERROR("missing .modinfo section"); 877 return NULL; 878 } 879 880 name = memmem(sec->data->d_buf, sec_size(sec), "\0name=", 6); 881 if (name) 882 return name + 6; 883 884 name = strdup(e->orig->name); 885 if (!name) { 886 ERROR_GLIBC("strdup"); 887 return NULL; 888 } 889 890 for (char *c = name; *c; c++) { 891 if (*c == '/') 892 name = c + 1; 893 else if (*c == '-') 894 *c = '_'; 895 else if (*c == '.') { 896 *c = '\0'; 897 break; 898 } 899 } 900 901 return name; 902 } 903 904 /* Get the object's module name as defined by the kernel (and klp_object) */ 905 static const char *find_modname(struct elfs *e) 906 { 907 const char *modname; 908 909 if (e->modname) 910 return e->modname; 911 912 modname = __find_modname(e); 913 e->modname = modname; 914 return modname; 915 } 916 917 /* 918 * Copying a function from its native compiled environment to a kernel module 919 * removes its natural access to local functions/variables and unexported 920 * globals. References to such symbols need to be converted to KLP relocs so 921 * the kernel arch relocation code knows to apply them and where to find the 922 * symbols. Particularly, duplicate static symbols need to be disambiguated. 923 */ 924 static bool klp_reloc_needed(struct reloc *patched_reloc) 925 { 926 struct symbol *patched_sym = patched_reloc->sym; 927 struct export *export; 928 929 /* no external symbol to reference */ 930 if (dont_correlate(patched_sym)) 931 return false; 932 933 /* For included functions, a regular reloc will do. */ 934 if (patched_sym->included) 935 return false; 936 937 /* 938 * If exported by a module, it has to be a klp reloc. Thanks to the 939 * clusterfunk that is late module patching, the patch module is 940 * allowed to be loaded before any modules it depends on. 941 * 942 * If exported by vmlinux, a normal reloc will do. 943 */ 944 export = find_export(patched_sym); 945 if (export) 946 return strcmp(export->mod, "vmlinux"); 947 948 if (!patched_sym->twin) { 949 /* 950 * Presumably the symbol and its reference were added by the 951 * patch. The symbol could be defined in this .o or in another 952 * .o in the patch module. 953 * 954 * This check needs to be *after* the export check due to the 955 * possibility of the patch adding a new UNDEF reference to an 956 * exported symbol. 957 */ 958 return false; 959 } 960 961 /* Unexported symbol which lives in the original vmlinux or module. */ 962 return true; 963 } 964 965 /* Return -1 error, 0 success, 1 skip */ 966 static int convert_reloc_sym_to_secsym(struct elf *elf, struct reloc *reloc) 967 { 968 struct symbol *sym = reloc->sym; 969 struct section *sec = sym->sec; 970 971 if (is_sec_sym(sym)) 972 return 0; 973 974 if (!sec->sym && !elf_create_section_symbol(elf, sec)) 975 return -1; 976 977 reloc->sym = sec->sym; 978 set_reloc_sym(elf, reloc, sec->sym->idx); 979 set_reloc_addend(elf, reloc, sym->offset + reloc_addend(reloc)); 980 return 0; 981 } 982 983 /* Return -1 error, 0 success, 1 skip */ 984 static int convert_reloc_secsym_to_sym(struct elf *elf, struct reloc *reloc) 985 { 986 struct symbol *sym = reloc->sym; 987 struct section *sec = sym->sec; 988 989 if (!is_sec_sym(sym)) 990 return 0; 991 992 /* If the symbol has a dedicated section, it's easy to find */ 993 sym = find_symbol_by_offset(sec, 0); 994 if (sym && sym->len == sec_size(sec)) 995 goto found_sym; 996 997 /* No dedicated section; find the symbol manually */ 998 sym = find_symbol_containing_inclusive(sec, arch_adjusted_addend(reloc)); 999 if (!sym) { 1000 /* 1001 * This is presumably an .altinstr_replacement section which is 1002 * empty due to it only having zero-length replacement(s). 1003 */ 1004 if (!sec_size(sec)) 1005 return 1; 1006 1007 /* 1008 * .rodata is a mixed bag of named objects and anonymous data. 1009 * 1010 * Convert section symbol references to named object symbols 1011 * when possible, to preserve pointer identity for const 1012 * structs like file_operations. Otherwise a section symbol is 1013 * fine. 1014 */ 1015 if (is_rodata_sec(sec)) 1016 return 0; 1017 1018 /* 1019 * This can happen for special section references to weak code 1020 * whose symbol has been stripped by the linker. 1021 */ 1022 return -1; 1023 } 1024 1025 found_sym: 1026 reloc->sym = sym; 1027 set_reloc_sym(elf, reloc, sym->idx); 1028 set_reloc_addend(elf, reloc, reloc_addend(reloc) - sym->offset); 1029 return 0; 1030 } 1031 1032 /* 1033 * Sections with anonymous or uncorrelated data (strings, UBSAN data, Clang 1034 * anonymous constants) need section symbol references. 1035 */ 1036 static bool is_uncorrelated_section(struct section *sec) 1037 { 1038 return is_string_sec(sec) || 1039 strstarts(sec->name, ".data..Lubsan") || /* GCC */ 1040 strstarts(sec->name, ".data..L__unnamed_") || /* Clang */ 1041 strstarts(sec->name, ".data..Lanon."); /* Clang */ 1042 } 1043 1044 /* 1045 * Convert a relocation symbol reference to the needed format: either a section 1046 * symbol or the underlying symbol itself. Return -1 error, 0 success, 1 skip. 1047 */ 1048 static int convert_reloc_sym(struct elf *elf, struct reloc *reloc) 1049 { 1050 struct section *sec = reloc->sym->sec; 1051 1052 if (reloc_type(reloc) == R_NONE) 1053 return 1; 1054 1055 if (is_uncorrelated_section(sec)) 1056 return convert_reloc_sym_to_secsym(elf, reloc); 1057 1058 /* Everything else: references should use named symbols. */ 1059 return convert_reloc_secsym_to_sym(elf, reloc); 1060 } 1061 1062 /* 1063 * Convert a regular relocation to a klp relocation (sort of). 1064 */ 1065 static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, 1066 struct section *sec, unsigned long offset, 1067 struct export *export) 1068 { 1069 struct symbol *patched_sym = patched_reloc->sym; 1070 s64 addend = reloc_addend(patched_reloc); 1071 const char *sym_modname, *sym_orig_name; 1072 static struct section *klp_relocs; 1073 struct symbol *sym, *klp_sym; 1074 unsigned long klp_reloc_off; 1075 char sym_name[SYM_NAME_LEN]; 1076 struct klp_reloc klp_reloc; 1077 unsigned long sympos; 1078 1079 if (!patched_sym->twin) { 1080 ERROR("unexpected klp reloc for new symbol %s", patched_sym->name); 1081 return -1; 1082 } 1083 1084 /* 1085 * Keep the original reloc intact for now to avoid breaking objtool run 1086 * which relies on proper relocations for many of its features. This 1087 * will be disabled later by "objtool klp post-link". 1088 * 1089 * Convert it to UNDEF (and WEAK to avoid modpost warnings). 1090 */ 1091 1092 sym = patched_sym->clone; 1093 if (!sym) { 1094 /* STB_WEAK: avoid modpost undefined symbol warnings */ 1095 sym = elf_create_symbol(e->out, patched_sym->name, NULL, 1096 STB_WEAK, patched_sym->type, 0, 0); 1097 if (!sym) 1098 return -1; 1099 1100 patched_sym->clone = sym; 1101 sym->clone = patched_sym; 1102 } 1103 1104 if (!elf_create_reloc(e->out, sec, offset, sym, addend, reloc_type(patched_reloc))) 1105 return -1; 1106 1107 /* 1108 * Create the KLP symbol. 1109 */ 1110 1111 if (export) { 1112 sym_modname = export->mod; 1113 sym_orig_name = export->sym; 1114 sympos = 0; 1115 } else { 1116 sym_modname = find_modname(e); 1117 if (!sym_modname) 1118 return -1; 1119 1120 sym_orig_name = patched_sym->twin->name; 1121 sympos = find_sympos(e->orig, patched_sym->twin); 1122 if (sympos == ULONG_MAX) 1123 return -1; 1124 } 1125 1126 /* symbol format: .klp.sym.modname.sym_name,sympos */ 1127 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_SYM_PREFIX "%s.%s,%ld", 1128 sym_modname, sym_orig_name, sympos)) 1129 return -1; 1130 1131 klp_sym = find_symbol_by_name(e->out, sym_name); 1132 if (!klp_sym) { 1133 __dbg_indent("%s", sym_name); 1134 1135 /* STB_WEAK: avoid modpost undefined symbol warnings */ 1136 klp_sym = elf_create_symbol(e->out, sym_name, NULL, 1137 STB_WEAK, patched_sym->type, 0, 0); 1138 if (!klp_sym) 1139 return -1; 1140 } 1141 1142 /* 1143 * Create the __klp_relocs entry. This will be converted to an actual 1144 * KLP rela by "objtool klp post-link". 1145 * 1146 * This intermediate step is necessary to prevent corruption by the 1147 * linker, which doesn't know how to properly handle two rela sections 1148 * applying to the same base section. 1149 */ 1150 1151 if (!klp_relocs) { 1152 klp_relocs = elf_create_section(e->out, KLP_RELOCS_SEC, 0, 1153 0, SHT_PROGBITS, 8, SHF_ALLOC); 1154 if (!klp_relocs) 1155 return -1; 1156 } 1157 1158 klp_reloc_off = sec_size(klp_relocs); 1159 memset(&klp_reloc, 0, sizeof(klp_reloc)); 1160 1161 klp_reloc.type = reloc_type(patched_reloc); 1162 if (!elf_add_data(e->out, klp_relocs, &klp_reloc, sizeof(klp_reloc))) 1163 return -1; 1164 1165 /* klp_reloc.offset */ 1166 if (!sec->sym && !elf_create_section_symbol(e->out, sec)) 1167 return -1; 1168 1169 if (!elf_create_reloc(e->out, klp_relocs, 1170 klp_reloc_off + offsetof(struct klp_reloc, offset), 1171 sec->sym, offset, R_ABS64)) 1172 return -1; 1173 1174 /* klp_reloc.sym */ 1175 if (!elf_create_reloc(e->out, klp_relocs, 1176 klp_reloc_off + offsetof(struct klp_reloc, sym), 1177 klp_sym, addend, R_ABS64)) 1178 return -1; 1179 1180 return 0; 1181 } 1182 1183 #define dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp) \ 1184 dbg_indent("%s+0x%lx: %s%s0x%lx [%s%s%s%s%s%s]", \ 1185 sec->name, offset, patched_sym->name, \ 1186 addend >= 0 ? "+" : "-", labs(addend), \ 1187 sym_type(patched_sym), \ 1188 is_sec_sym(patched_sym) ? "" : " ", \ 1189 is_sec_sym(patched_sym) ? "" : sym_bind(patched_sym), \ 1190 is_undef_sym(patched_sym) ? " UNDEF" : "", \ 1191 export ? " EXPORTED" : "", \ 1192 klp ? " KLP" : "") 1193 1194 /* Copy a reloc and its symbol to the output object */ 1195 static int clone_reloc(struct elfs *e, struct reloc *patched_reloc, 1196 struct section *sec, unsigned long offset) 1197 { 1198 struct symbol *patched_sym = patched_reloc->sym; 1199 struct export *export = find_export(patched_sym); 1200 long addend = reloc_addend(patched_reloc); 1201 struct symbol *out_sym; 1202 bool klp; 1203 1204 klp = klp_reloc_needed(patched_reloc); 1205 1206 dbg_clone_reloc(sec, offset, patched_sym, addend, export, klp); 1207 1208 if (klp) { 1209 if (clone_reloc_klp(e, patched_reloc, sec, offset, export)) 1210 return -1; 1211 1212 return 0; 1213 } 1214 1215 /* 1216 * Why !export sets 'data_too': 1217 * 1218 * Unexported non-klp symbols need to live in the patch module, 1219 * otherwise there will be unresolved symbols. Notably, this includes: 1220 * 1221 * - New functions/data 1222 * - String sections 1223 * - Special section entries 1224 * - Uncorrelated static local variables 1225 * - UBSAN sections 1226 */ 1227 out_sym = clone_symbol(e, patched_sym, patched_sym->included || !export); 1228 if (!out_sym) 1229 return -1; 1230 1231 /* 1232 * For strings, all references use section symbols, thanks to 1233 * convert_reloc_sym(). clone_symbol() has cloned an empty 1234 * version of the string section. Now copy the string itself. 1235 */ 1236 if (is_string_sec(patched_sym->sec)) { 1237 const char *str = patched_sym->sec->data->d_buf + addend; 1238 1239 __dbg_indent("\"%s\"", escape_str(str)); 1240 1241 addend = elf_add_string(e->out, out_sym->sec, str); 1242 if (addend == -1) 1243 return -1; 1244 } 1245 1246 if (!elf_create_reloc(e->out, sec, offset, out_sym, addend, 1247 reloc_type(patched_reloc))) 1248 return -1; 1249 1250 return 0; 1251 } 1252 1253 /* Copy all relocs needed for a symbol's contents */ 1254 static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym) 1255 { 1256 struct section *patched_rsec = patched_sym->sec->rsec; 1257 struct reloc *patched_reloc; 1258 unsigned long start, end; 1259 struct symbol *out_sym; 1260 1261 out_sym = patched_sym->clone; 1262 if (!out_sym) { 1263 ERROR("no clone for %s", patched_sym->name); 1264 return -1; 1265 } 1266 1267 if (!patched_rsec) 1268 return 0; 1269 1270 if (!is_sec_sym(patched_sym) && !patched_sym->len) 1271 return 0; 1272 1273 if (is_string_sec(patched_sym->sec)) 1274 return 0; 1275 1276 if (is_sec_sym(patched_sym)) { 1277 start = 0; 1278 end = sec_size(patched_sym->sec); 1279 } else { 1280 start = patched_sym->offset; 1281 end = start + patched_sym->len; 1282 } 1283 1284 for_each_reloc(patched_rsec, patched_reloc) { 1285 unsigned long offset; 1286 int ret; 1287 1288 if (reloc_offset(patched_reloc) < start || 1289 reloc_offset(patched_reloc) >= end) 1290 continue; 1291 1292 /* 1293 * Skip any reloc referencing .altinstr_aux. Its code is 1294 * always patched by alternatives. See ALTERNATIVE_TERNARY(). 1295 */ 1296 if (patched_reloc->sym->sec && 1297 !strcmp(patched_reloc->sym->sec->name, ".altinstr_aux")) 1298 continue; 1299 1300 ret = convert_reloc_sym(e->patched, patched_reloc); 1301 if (ret < 0) { 1302 ERROR_FUNC(patched_rsec->base, reloc_offset(patched_reloc), 1303 "failed to convert reloc sym '%s' to its proper format", 1304 patched_reloc->sym->name); 1305 return -1; 1306 } 1307 if (ret > 0) 1308 continue; 1309 1310 offset = out_sym->offset + (reloc_offset(patched_reloc) - patched_sym->offset); 1311 1312 if (clone_reloc(e, patched_reloc, out_sym->sec, offset)) 1313 return -1; 1314 } 1315 return 0; 1316 1317 } 1318 1319 static int create_fake_symbol(struct elf *elf, struct section *sec, 1320 unsigned long offset, size_t size) 1321 { 1322 char name[SYM_NAME_LEN]; 1323 unsigned int type; 1324 static int ctr; 1325 char *c; 1326 1327 if (snprintf_check(name, SYM_NAME_LEN, "%s_%d", sec->name, ctr++)) 1328 return -1; 1329 1330 for (c = name; *c; c++) 1331 if (*c == '.') 1332 *c = '_'; 1333 1334 /* 1335 * STT_NOTYPE: Prevent objtool from validating .altinstr_replacement 1336 * while still allowing objdump to disassemble it. 1337 */ 1338 type = is_text_sec(sec) ? STT_NOTYPE : STT_OBJECT; 1339 return elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size) ? 0 : -1; 1340 } 1341 1342 /* 1343 * Special sections (alternatives, etc) are basically arrays of structs. 1344 * For all the special sections, create a symbol for each struct entry. This 1345 * is a bit cumbersome, but it makes the extracting of the individual entries 1346 * much more straightforward. 1347 * 1348 * There are three ways to identify the entry sizes for a special section: 1349 * 1350 * 1) ELF section header sh_entsize: Ideally this would be used almost 1351 * everywhere. But unfortunately the toolchains make it difficult. The 1352 * assembler .[push]section directive syntax only takes entsize when 1353 * combined with SHF_MERGE. But Clang disallows combining SHF_MERGE with 1354 * SHF_WRITE. And some special sections do need to be writable. 1355 * 1356 * Another place this wouldn't work is .altinstr_replacement, whose entries 1357 * don't have a fixed size. 1358 * 1359 * 2) ANNOTATE_DATA_SPECIAL: This is a lightweight objtool annotation which 1360 * points to the beginning of each entry. The size of the entry is then 1361 * inferred by the location of the subsequent annotation (or end of 1362 * section). 1363 * 1364 * 3) Simple array of pointers: If the special section is just a basic array of 1365 * pointers, the entry size can be inferred by the number of relocations. 1366 * No annotations needed. 1367 * 1368 * Note I also tried to create per-entry symbols at the time of creation, in 1369 * the original [inline] asm. Unfortunately, creating uniquely named symbols 1370 * is trickier than one might think, especially with Clang inline asm. I 1371 * eventually just gave up trying to make that work, in favor of using 1372 * ANNOTATE_DATA_SPECIAL and creating the symbols here after the fact. 1373 */ 1374 static int create_fake_symbols(struct elf *elf) 1375 { 1376 struct section *sec; 1377 struct reloc *reloc; 1378 1379 /* 1380 * 1) Make symbols for all the ANNOTATE_DATA_SPECIAL entries: 1381 */ 1382 1383 sec = find_section_by_name(elf, ".discard.annotate_data"); 1384 if (!sec || !sec->rsec) 1385 goto entsize; 1386 1387 for_each_reloc(sec->rsec, reloc) { 1388 unsigned long offset, size; 1389 struct reloc *next_reloc; 1390 1391 if (annotype(elf, sec, reloc) != ANNOTYPE_DATA_SPECIAL) 1392 continue; 1393 1394 offset = reloc_addend(reloc); 1395 1396 size = 0; 1397 next_reloc = reloc; 1398 for_each_reloc_continue(sec->rsec, next_reloc) { 1399 if (annotype(elf, sec, next_reloc) != ANNOTYPE_DATA_SPECIAL || 1400 next_reloc->sym->sec != reloc->sym->sec) 1401 continue; 1402 1403 size = reloc_addend(next_reloc) - offset; 1404 break; 1405 } 1406 1407 if (!size) 1408 size = sec_size(reloc->sym->sec) - offset; 1409 1410 if (create_fake_symbol(elf, reloc->sym->sec, offset, size)) 1411 return -1; 1412 } 1413 1414 /* 1415 * 2) Make symbols for sh_entsize, and simple arrays of pointers: 1416 */ 1417 entsize: 1418 for_each_sec(elf, sec) { 1419 unsigned int entry_size; 1420 unsigned long offset; 1421 1422 if (!is_special_section(sec) || find_symbol_by_offset(sec, 0)) 1423 continue; 1424 1425 if (!sec->rsec) { 1426 ERROR("%s: missing special section relocations", sec->name); 1427 return -1; 1428 } 1429 1430 entry_size = sec->sh.sh_entsize; 1431 if (!entry_size) { 1432 entry_size = arch_reloc_size(sec->rsec->relocs); 1433 if (sec_size(sec) != entry_size * sec_num_entries(sec->rsec)) { 1434 ERROR("%s: missing special section entsize or annotations", sec->name); 1435 return -1; 1436 } 1437 } 1438 1439 for (offset = 0; offset < sec_size(sec); offset += entry_size) { 1440 if (create_fake_symbol(elf, sec, offset, entry_size)) 1441 return -1; 1442 } 1443 } 1444 1445 return 0; 1446 } 1447 1448 /* Keep a special section entry if it references an included function */ 1449 static bool should_keep_special_sym(struct elf *elf, struct symbol *sym) 1450 { 1451 bool annotate_insn = !strcmp(sym->sec->name, ".discard.annotate_insn"); 1452 struct reloc *reloc; 1453 1454 if (is_sec_sym(sym) || !sym->sec->rsec) 1455 return false; 1456 1457 sym_for_each_reloc(elf, sym, reloc) { 1458 if (convert_reloc_sym(elf, reloc)) 1459 continue; 1460 1461 if (!reloc->sym->clone || is_undef_sym(reloc->sym->clone)) 1462 continue; 1463 1464 /* 1465 * Keep special section references to cloned functions. 1466 * In some cases annotate_insn can also reference cloned alt 1467 * replacement fake symbols; keep those references as well. 1468 */ 1469 if (is_func_sym(reloc->sym) || 1470 (annotate_insn && is_notype_sym(reloc->sym))) 1471 return true; 1472 } 1473 1474 return false; 1475 } 1476 1477 /* 1478 * Klp relocations aren't allowed for __jump_table and .static_call_sites if 1479 * the referenced symbol lives in a kernel module, because such klp relocs may 1480 * be applied after static branch/call init, resulting in code corruption. 1481 * 1482 * Validate a special section entry to avoid that. Note that an inert 1483 * tracepoint or pr_debug() is harmless enough, in that case just skip the 1484 * entry and print a warning. Otherwise, return an error. 1485 * 1486 * TODO: This is only a temporary limitation which will be fixed when livepatch 1487 * adds support for submodules: fully self-contained modules which are embedded 1488 * in the top-level livepatch module's data and which can be loaded on demand 1489 * when their corresponding to-be-patched module gets loaded. Then klp relocs 1490 * can be retired. 1491 * 1492 * Return: 1493 * -1: error: validation failed 1494 * 1: warning: disabled tracepoint or pr_debug() 1495 * 0: success 1496 */ 1497 static int validate_special_section_klp_reloc(struct elfs *e, struct symbol *sym) 1498 { 1499 bool static_branch = !strcmp(sym->sec->name, "__jump_table"); 1500 bool static_call = !strcmp(sym->sec->name, ".static_call_sites"); 1501 const char *code_sym = NULL; 1502 unsigned long code_offset = 0; 1503 struct reloc *reloc; 1504 int ret = 0; 1505 1506 if (!static_branch && !static_call) 1507 return 0; 1508 1509 sym_for_each_reloc(e->patched, sym, reloc) { 1510 const char *sym_modname; 1511 struct export *export; 1512 1513 if (convert_reloc_sym(e->patched, reloc)) 1514 continue; 1515 1516 /* Static branch/call keys are always STT_OBJECT */ 1517 if (reloc->sym->type != STT_OBJECT) { 1518 1519 /* Save code location which can be printed below */ 1520 if (reloc->sym->type == STT_FUNC && !code_sym) { 1521 code_sym = reloc->sym->name; 1522 code_offset = reloc_addend(reloc); 1523 } 1524 1525 continue; 1526 } 1527 1528 if (!klp_reloc_needed(reloc)) 1529 continue; 1530 1531 export = find_export(reloc->sym); 1532 if (export) { 1533 sym_modname = export->mod; 1534 } else { 1535 sym_modname = find_modname(e); 1536 if (!sym_modname) 1537 return -1; 1538 } 1539 1540 /* vmlinux keys are ok */ 1541 if (!strcmp(sym_modname, "vmlinux")) 1542 continue; 1543 1544 if (!code_sym) 1545 code_sym = "<unknown>"; 1546 1547 if (static_branch) { 1548 if (strstarts(reloc->sym->name, "__tracepoint_")) { 1549 WARN("%s: disabling unsupported tracepoint %s", 1550 code_sym, reloc->sym->name + 13); 1551 ret = 1; 1552 continue; 1553 } 1554 1555 if (strstr(reloc->sym->name, "__UNIQUE_ID_ddebug_")) { 1556 WARN("%s: disabling unsupported pr_debug()", 1557 code_sym); 1558 ret = 1; 1559 continue; 1560 } 1561 1562 ERROR("%s+0x%lx: unsupported static branch key %s. Use static_key_enabled() instead", 1563 code_sym, code_offset, reloc->sym->name); 1564 return -1; 1565 } 1566 1567 /* static call */ 1568 if (strstarts(reloc->sym->name, "__SCK__tp_func_")) { 1569 ret = 1; 1570 continue; 1571 } 1572 1573 ERROR("%s()+0x%lx: unsupported static call key %s. Use KLP_STATIC_CALL() instead", 1574 code_sym, code_offset, reloc->sym->name); 1575 return -1; 1576 } 1577 1578 return ret; 1579 } 1580 1581 static int clone_special_section(struct elfs *e, struct section *patched_sec) 1582 { 1583 struct symbol *patched_sym; 1584 1585 /* 1586 * Extract all special section symbols (and their dependencies) which 1587 * reference included functions. 1588 */ 1589 sec_for_each_sym(patched_sec, patched_sym) { 1590 int ret; 1591 1592 if (!is_object_sym(patched_sym)) 1593 continue; 1594 1595 if (!should_keep_special_sym(e->patched, patched_sym)) 1596 continue; 1597 1598 ret = validate_special_section_klp_reloc(e, patched_sym); 1599 if (ret < 0) 1600 return -1; 1601 if (ret > 0) 1602 continue; 1603 1604 if (!clone_symbol(e, patched_sym, true)) 1605 return -1; 1606 } 1607 1608 return 0; 1609 } 1610 1611 /* Extract only the needed bits from special sections */ 1612 static int clone_special_sections(struct elfs *e) 1613 { 1614 struct section *sec, *annotate_insn = NULL; 1615 1616 for_each_sec(e->patched, sec) { 1617 if (is_special_section(sec)) { 1618 if (!strcmp(sec->name, ".discard.annotate_insn")) { 1619 annotate_insn = sec; 1620 continue; 1621 } 1622 if (clone_special_section(e, sec)) 1623 return -1; 1624 } 1625 } 1626 1627 /* 1628 * Do .discard.annotate_insn last, it can reference other special 1629 * sections (alt replacements) so they need to be cloned first. 1630 */ 1631 if (annotate_insn) { 1632 if (clone_special_section(e, annotate_insn)) 1633 return -1; 1634 } 1635 1636 return 0; 1637 } 1638 1639 /* 1640 * Create .init.klp_objects and .init.klp_funcs sections which are intermediate 1641 * sections provided as input to the patch module's init code for building the 1642 * klp_patch, klp_object and klp_func structs for the livepatch API. 1643 */ 1644 static int create_klp_sections(struct elfs *e) 1645 { 1646 size_t obj_size = sizeof(struct klp_object_ext); 1647 size_t func_size = sizeof(struct klp_func_ext); 1648 struct section *obj_sec, *funcs_sec, *str_sec; 1649 struct symbol *funcs_sym, *str_sym, *sym; 1650 char sym_name[SYM_NAME_LEN]; 1651 unsigned int nr_funcs = 0; 1652 const char *modname; 1653 void *obj_data; 1654 s64 addend; 1655 1656 obj_sec = elf_create_section_pair(e->out, KLP_OBJECTS_SEC, obj_size, 0, 0); 1657 if (!obj_sec) 1658 return -1; 1659 1660 funcs_sec = elf_create_section_pair(e->out, KLP_FUNCS_SEC, func_size, 0, 0); 1661 if (!funcs_sec) 1662 return -1; 1663 1664 funcs_sym = elf_create_section_symbol(e->out, funcs_sec); 1665 if (!funcs_sym) 1666 return -1; 1667 1668 str_sec = elf_create_section(e->out, KLP_STRINGS_SEC, 0, 0, 1669 SHT_PROGBITS, 1, 1670 SHF_ALLOC | SHF_STRINGS | SHF_MERGE); 1671 if (!str_sec) 1672 return -1; 1673 1674 if (elf_add_string(e->out, str_sec, "") == -1) 1675 return -1; 1676 1677 str_sym = elf_create_section_symbol(e->out, str_sec); 1678 if (!str_sym) 1679 return -1; 1680 1681 /* allocate klp_object_ext */ 1682 obj_data = elf_add_data(e->out, obj_sec, NULL, obj_size); 1683 if (!obj_data) 1684 return -1; 1685 1686 modname = find_modname(e); 1687 if (!modname) 1688 return -1; 1689 1690 /* klp_object_ext.name */ 1691 if (strcmp(modname, "vmlinux")) { 1692 addend = elf_add_string(e->out, str_sec, modname); 1693 if (addend == -1) 1694 return -1; 1695 1696 if (!elf_create_reloc(e->out, obj_sec, 1697 offsetof(struct klp_object_ext, name), 1698 str_sym, addend, R_ABS64)) 1699 return -1; 1700 } 1701 1702 /* klp_object_ext.funcs */ 1703 if (!elf_create_reloc(e->out, obj_sec, offsetof(struct klp_object_ext, funcs), 1704 funcs_sym, 0, R_ABS64)) 1705 return -1; 1706 1707 for_each_sym(e->out, sym) { 1708 unsigned long offset = nr_funcs * func_size; 1709 unsigned long sympos; 1710 void *func_data; 1711 1712 if (!is_func_sym(sym) || sym->cold || !sym->clone || !sym->clone->changed) 1713 continue; 1714 1715 /* allocate klp_func_ext */ 1716 func_data = elf_add_data(e->out, funcs_sec, NULL, func_size); 1717 if (!func_data) 1718 return -1; 1719 1720 /* klp_func_ext.old_name */ 1721 addend = elf_add_string(e->out, str_sec, sym->clone->twin->name); 1722 if (addend == -1) 1723 return -1; 1724 1725 if (!elf_create_reloc(e->out, funcs_sec, 1726 offset + offsetof(struct klp_func_ext, old_name), 1727 str_sym, addend, R_ABS64)) 1728 return -1; 1729 1730 /* klp_func_ext.new_func */ 1731 if (!elf_create_reloc(e->out, funcs_sec, 1732 offset + offsetof(struct klp_func_ext, new_func), 1733 sym, 0, R_ABS64)) 1734 return -1; 1735 1736 /* klp_func_ext.sympos */ 1737 BUILD_BUG_ON(sizeof(sympos) != sizeof_field(struct klp_func_ext, sympos)); 1738 sympos = find_sympos(e->orig, sym->clone->twin); 1739 if (sympos == ULONG_MAX) 1740 return -1; 1741 memcpy(func_data + offsetof(struct klp_func_ext, sympos), &sympos, 1742 sizeof_field(struct klp_func_ext, sympos)); 1743 1744 nr_funcs++; 1745 } 1746 1747 /* klp_object_ext.nr_funcs */ 1748 BUILD_BUG_ON(sizeof(nr_funcs) != sizeof_field(struct klp_object_ext, nr_funcs)); 1749 memcpy(obj_data + offsetof(struct klp_object_ext, nr_funcs), &nr_funcs, 1750 sizeof_field(struct klp_object_ext, nr_funcs)); 1751 1752 /* 1753 * Find callback pointers created by KLP_PRE_PATCH_CALLBACK() and 1754 * friends, and add them to the klp object. 1755 */ 1756 1757 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_PRE_PATCH_PREFIX "%s", modname)) 1758 return -1; 1759 1760 sym = find_symbol_by_name(e->out, sym_name); 1761 if (sym) { 1762 struct reloc *reloc; 1763 1764 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset); 1765 1766 if (!elf_create_reloc(e->out, obj_sec, 1767 offsetof(struct klp_object_ext, callbacks) + 1768 offsetof(struct klp_callbacks, pre_patch), 1769 reloc->sym, reloc_addend(reloc), R_ABS64)) 1770 return -1; 1771 } 1772 1773 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_POST_PATCH_PREFIX "%s", modname)) 1774 return -1; 1775 1776 sym = find_symbol_by_name(e->out, sym_name); 1777 if (sym) { 1778 struct reloc *reloc; 1779 1780 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset); 1781 1782 if (!elf_create_reloc(e->out, obj_sec, 1783 offsetof(struct klp_object_ext, callbacks) + 1784 offsetof(struct klp_callbacks, post_patch), 1785 reloc->sym, reloc_addend(reloc), R_ABS64)) 1786 return -1; 1787 } 1788 1789 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_PRE_UNPATCH_PREFIX "%s", modname)) 1790 return -1; 1791 1792 sym = find_symbol_by_name(e->out, sym_name); 1793 if (sym) { 1794 struct reloc *reloc; 1795 1796 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset); 1797 1798 if (!elf_create_reloc(e->out, obj_sec, 1799 offsetof(struct klp_object_ext, callbacks) + 1800 offsetof(struct klp_callbacks, pre_unpatch), 1801 reloc->sym, reloc_addend(reloc), R_ABS64)) 1802 return -1; 1803 } 1804 1805 if (snprintf_check(sym_name, SYM_NAME_LEN, KLP_POST_UNPATCH_PREFIX "%s", modname)) 1806 return -1; 1807 1808 sym = find_symbol_by_name(e->out, sym_name); 1809 if (sym) { 1810 struct reloc *reloc; 1811 1812 reloc = find_reloc_by_dest(e->out, sym->sec, sym->offset); 1813 1814 if (!elf_create_reloc(e->out, obj_sec, 1815 offsetof(struct klp_object_ext, callbacks) + 1816 offsetof(struct klp_callbacks, post_unpatch), 1817 reloc->sym, reloc_addend(reloc), R_ABS64)) 1818 return -1; 1819 } 1820 1821 return 0; 1822 } 1823 1824 /* 1825 * Copy all .modinfo import_ns= tags to ensure all namespaced exported symbols 1826 * can be accessed via normal relocs. 1827 */ 1828 static int copy_import_ns(struct elfs *e) 1829 { 1830 struct section *patched_sec, *out_sec = NULL; 1831 char *import_ns, *data_end; 1832 1833 patched_sec = find_section_by_name(e->patched, ".modinfo"); 1834 if (!patched_sec) 1835 return 0; 1836 1837 import_ns = patched_sec->data->d_buf; 1838 if (!import_ns) 1839 return 0; 1840 1841 for (data_end = import_ns + sec_size(patched_sec); 1842 import_ns < data_end; 1843 import_ns += strlen(import_ns) + 1) { 1844 1845 import_ns = memmem(import_ns, data_end - import_ns, "import_ns=", 10); 1846 if (!import_ns) 1847 return 0; 1848 1849 if (!out_sec) { 1850 out_sec = find_section_by_name(e->out, ".modinfo"); 1851 if (!out_sec) { 1852 out_sec = elf_create_section(e->out, ".modinfo", 0, 1853 patched_sec->sh.sh_entsize, 1854 patched_sec->sh.sh_type, 1855 patched_sec->sh.sh_addralign, 1856 patched_sec->sh.sh_flags); 1857 if (!out_sec) 1858 return -1; 1859 } 1860 } 1861 1862 if (!elf_add_data(e->out, out_sec, import_ns, strlen(import_ns) + 1)) 1863 return -1; 1864 } 1865 1866 return 0; 1867 } 1868 1869 int cmd_klp_diff(int argc, const char **argv) 1870 { 1871 struct elfs e = {0}; 1872 1873 argc = parse_options(argc, argv, klp_diff_options, klp_diff_usage, 0); 1874 if (argc != 3) 1875 usage_with_options(klp_diff_usage, klp_diff_options); 1876 1877 objname = argv[0]; 1878 1879 e.orig = elf_open_read(argv[0], O_RDONLY); 1880 e.patched = elf_open_read(argv[1], O_RDONLY); 1881 e.out = NULL; 1882 1883 if (!e.orig || !e.patched) 1884 return -1; 1885 1886 if (read_exports()) 1887 return -1; 1888 1889 if (read_sym_checksums(e.orig)) 1890 return -1; 1891 1892 if (read_sym_checksums(e.patched)) 1893 return -1; 1894 1895 if (correlate_symbols(&e)) 1896 return -1; 1897 1898 if (mark_changed_functions(&e)) 1899 return 0; 1900 1901 e.out = elf_create_file(&e.orig->ehdr, argv[2]); 1902 if (!e.out) 1903 return -1; 1904 1905 /* 1906 * Special section fake symbols are needed so that individual special 1907 * section entries can be extracted by clone_special_sections(). 1908 * 1909 * Note the fake symbols are also needed by clone_included_functions() 1910 * because __WARN_printf() call sites add references to bug table 1911 * entries in the calling functions. 1912 */ 1913 if (create_fake_symbols(e.patched)) 1914 return -1; 1915 1916 if (clone_included_functions(&e)) 1917 return -1; 1918 1919 if (clone_special_sections(&e)) 1920 return -1; 1921 1922 if (create_klp_sections(&e)) 1923 return -1; 1924 1925 if (copy_import_ns(&e)) 1926 return -1; 1927 1928 if (elf_write(e.out)) 1929 return -1; 1930 1931 return elf_close(e.out); 1932 } 1933