1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * sorttable.c: Sort the kernel's table 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <shile.zhang@linux.alibaba.com> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Based on code taken from recortmcount.c which is: 12 * 13 * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved. 14 * 15 * Restructured to fit Linux format, as well as other updates: 16 * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc. 17 */ 18 19 /* 20 * Strategy: alter the vmlinux file in-place. 21 */ 22 23 #include <sys/types.h> 24 #include <sys/stat.h> 25 #include <getopt.h> 26 #include <fcntl.h> 27 #include <stdio.h> 28 #include <stdlib.h> 29 #include <stdbool.h> 30 #include <string.h> 31 #include <unistd.h> 32 #include <errno.h> 33 #include <pthread.h> 34 35 #include "elf-parse.h" 36 37 #ifndef EM_ARCOMPACT 38 #define EM_ARCOMPACT 93 39 #endif 40 41 #ifndef EM_XTENSA 42 #define EM_XTENSA 94 43 #endif 44 45 #ifndef EM_AARCH64 46 #define EM_AARCH64 183 47 #endif 48 49 #ifndef EM_MICROBLAZE 50 #define EM_MICROBLAZE 189 51 #endif 52 53 #ifndef EM_ARCV2 54 #define EM_ARCV2 195 55 #endif 56 57 #ifndef EM_RISCV 58 #define EM_RISCV 243 59 #endif 60 61 #ifndef EM_LOONGARCH 62 #define EM_LOONGARCH 258 63 #endif 64 65 typedef void (*table_sort_t)(char *, int); 66 67 /* 68 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of 69 * the way to -256..-1, to avoid conflicting with real section 70 * indices. 71 */ 72 #define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1)) 73 74 static inline int is_shndx_special(unsigned int i) 75 { 76 return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE; 77 } 78 79 /* Accessor for sym->st_shndx, hides ugliness of "64k sections" */ 80 static inline unsigned int get_secindex(unsigned int shndx, 81 unsigned int sym_offs, 82 const Elf32_Word *symtab_shndx_start) 83 { 84 if (is_shndx_special(shndx)) 85 return SPECIAL(shndx); 86 if (shndx != SHN_XINDEX) 87 return shndx; 88 return elf_parser.r(&symtab_shndx_start[sym_offs]); 89 } 90 91 static int compare_extable_32(const void *a, const void *b) 92 { 93 Elf32_Addr av = elf_parser.r(a); 94 Elf32_Addr bv = elf_parser.r(b); 95 96 if (av < bv) 97 return -1; 98 return av > bv; 99 } 100 101 static int compare_extable_64(const void *a, const void *b) 102 { 103 Elf64_Addr av = elf_parser.r8(a); 104 Elf64_Addr bv = elf_parser.r8(b); 105 106 if (av < bv) 107 return -1; 108 return av > bv; 109 } 110 111 static int (*compare_extable)(const void *a, const void *b); 112 113 static inline void *get_index(void *start, int entsize, int index) 114 { 115 return start + (entsize * index); 116 } 117 118 static int extable_ent_size; 119 static int long_size; 120 121 #define ERRSTR_MAXSZ 256 122 123 #ifdef UNWINDER_ORC_ENABLED 124 /* ORC unwinder only support X86_64 */ 125 #include <asm/orc_types.h> 126 127 static char g_err[ERRSTR_MAXSZ]; 128 static int *g_orc_ip_table; 129 static struct orc_entry *g_orc_table; 130 131 static pthread_t orc_sort_thread; 132 133 static inline unsigned long orc_ip(const int *ip) 134 { 135 return (unsigned long)ip + *ip; 136 } 137 138 static int orc_sort_cmp(const void *_a, const void *_b) 139 { 140 struct orc_entry *orc_a, *orc_b; 141 const int *a = g_orc_ip_table + *(int *)_a; 142 const int *b = g_orc_ip_table + *(int *)_b; 143 unsigned long a_val = orc_ip(a); 144 unsigned long b_val = orc_ip(b); 145 146 if (a_val > b_val) 147 return 1; 148 if (a_val < b_val) 149 return -1; 150 151 /* 152 * The "weak" section terminator entries need to always be on the left 153 * to ensure the lookup code skips them in favor of real entries. 154 * These terminator entries exist to handle any gaps created by 155 * whitelisted .o files which didn't get objtool generation. 156 */ 157 orc_a = g_orc_table + (a - g_orc_ip_table); 158 orc_b = g_orc_table + (b - g_orc_ip_table); 159 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED) 160 return 0; 161 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 162 } 163 164 static void *sort_orctable(void *arg) 165 { 166 int i; 167 int *idxs = NULL; 168 int *tmp_orc_ip_table = NULL; 169 struct orc_entry *tmp_orc_table = NULL; 170 unsigned int *orc_ip_size = (unsigned int *)arg; 171 unsigned int num_entries = *orc_ip_size / sizeof(int); 172 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 173 174 idxs = (int *)malloc(*orc_ip_size); 175 if (!idxs) { 176 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 177 strerror(errno)); 178 pthread_exit(g_err); 179 } 180 181 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 182 if (!tmp_orc_ip_table) { 183 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 184 strerror(errno)); 185 pthread_exit(g_err); 186 } 187 188 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 189 if (!tmp_orc_table) { 190 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 191 strerror(errno)); 192 pthread_exit(g_err); 193 } 194 195 /* initialize indices array, convert ip_table to absolute address */ 196 for (i = 0; i < num_entries; i++) { 197 idxs[i] = i; 198 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 199 } 200 memcpy(tmp_orc_table, g_orc_table, orc_size); 201 202 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 203 204 for (i = 0; i < num_entries; i++) { 205 if (idxs[i] == i) 206 continue; 207 208 /* convert back to relative address */ 209 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 210 g_orc_table[i] = tmp_orc_table[idxs[i]]; 211 } 212 213 free(idxs); 214 free(tmp_orc_ip_table); 215 free(tmp_orc_table); 216 pthread_exit(NULL); 217 } 218 #endif 219 220 #ifdef MCOUNT_SORT_ENABLED 221 222 static int compare_values_64(const void *a, const void *b) 223 { 224 uint64_t av = *(uint64_t *)a; 225 uint64_t bv = *(uint64_t *)b; 226 227 if (av < bv) 228 return -1; 229 return av > bv; 230 } 231 232 static int compare_values_32(const void *a, const void *b) 233 { 234 uint32_t av = *(uint32_t *)a; 235 uint32_t bv = *(uint32_t *)b; 236 237 if (av < bv) 238 return -1; 239 return av > bv; 240 } 241 242 static int (*compare_values)(const void *a, const void *b); 243 244 /* Only used for sorting mcount table */ 245 static void rela_write_addend(Elf_Rela *rela, uint64_t val) 246 { 247 elf_parser.rela_write_addend(rela, val); 248 } 249 250 struct func_info { 251 uint64_t addr; 252 uint64_t size; 253 }; 254 255 /* List of functions created by: nm -S vmlinux */ 256 static struct func_info *function_list; 257 static int function_list_size; 258 259 /* Allocate functions in 1k blocks */ 260 #define FUNC_BLK_SIZE 1024 261 #define FUNC_BLK_MASK (FUNC_BLK_SIZE - 1) 262 263 static int add_field(uint64_t addr, uint64_t size) 264 { 265 struct func_info *fi; 266 int fsize = function_list_size; 267 268 if (!(fsize & FUNC_BLK_MASK)) { 269 fsize += FUNC_BLK_SIZE; 270 fi = realloc(function_list, fsize * sizeof(struct func_info)); 271 if (!fi) 272 return -1; 273 function_list = fi; 274 } 275 fi = &function_list[function_list_size++]; 276 fi->addr = addr; 277 fi->size = size; 278 return 0; 279 } 280 281 /* Used for when mcount/fentry is before the function entry */ 282 static int before_func; 283 284 /* Only return match if the address lies inside the function size */ 285 static int cmp_func_addr(const void *K, const void *A) 286 { 287 uint64_t key = *(const uint64_t *)K; 288 const struct func_info *a = A; 289 290 if (key + before_func < a->addr) 291 return -1; 292 return key >= a->addr + a->size; 293 } 294 295 /* Find the function in function list that is bounded by the function size */ 296 static int find_func(uint64_t key) 297 { 298 return bsearch(&key, function_list, function_list_size, 299 sizeof(struct func_info), cmp_func_addr) != NULL; 300 } 301 302 static int cmp_funcs(const void *A, const void *B) 303 { 304 const struct func_info *a = A; 305 const struct func_info *b = B; 306 307 if (a->addr < b->addr) 308 return -1; 309 return a->addr > b->addr; 310 } 311 312 static int parse_symbols(const char *fname) 313 { 314 FILE *fp; 315 char addr_str[20]; /* Only need 17, but round up to next int size */ 316 char size_str[20]; 317 char type; 318 319 fp = fopen(fname, "r"); 320 if (!fp) { 321 perror(fname); 322 return -1; 323 } 324 325 while (fscanf(fp, "%16s %16s %c %*s\n", addr_str, size_str, &type) == 3) { 326 uint64_t addr; 327 uint64_t size; 328 329 /* Only care about functions */ 330 if (type != 't' && type != 'T' && type != 'W') 331 continue; 332 333 addr = strtoull(addr_str, NULL, 16); 334 size = strtoull(size_str, NULL, 16); 335 if (add_field(addr, size) < 0) 336 return -1; 337 } 338 fclose(fp); 339 340 qsort(function_list, function_list_size, sizeof(struct func_info), cmp_funcs); 341 342 return 0; 343 } 344 345 static pthread_t mcount_sort_thread; 346 static bool sort_reloc; 347 348 static long rela_type; 349 350 static char m_err[ERRSTR_MAXSZ]; 351 352 struct elf_mcount_loc { 353 Elf_Ehdr *ehdr; 354 Elf_Shdr *init_data_sec; 355 uint64_t start_mcount_loc; 356 uint64_t stop_mcount_loc; 357 }; 358 359 /* Fill the array with the content of the relocs */ 360 static int fill_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 361 { 362 Elf_Shdr *shdr_start; 363 Elf_Rela *rel; 364 unsigned int shnum; 365 unsigned int count = 0; 366 int shentsize; 367 void *array_end = ptr + size; 368 369 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 370 shentsize = ehdr_shentsize(ehdr); 371 372 shnum = ehdr_shnum(ehdr); 373 if (shnum == SHN_UNDEF) 374 shnum = shdr_size(shdr_start); 375 376 for (int i = 0; i < shnum; i++) { 377 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 378 void *end; 379 380 if (shdr_type(shdr) != SHT_RELA) 381 continue; 382 383 rel = (void *)ehdr + shdr_offset(shdr); 384 end = (void *)rel + shdr_size(shdr); 385 386 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 387 uint64_t offset = rela_offset(rel); 388 389 if (offset >= start_loc && offset < start_loc + size) { 390 if (ptr + long_size > array_end) { 391 snprintf(m_err, ERRSTR_MAXSZ, 392 "Too many relocations"); 393 return -1; 394 } 395 396 /* Make sure this has the correct type */ 397 if (rela_info(rel) != rela_type) { 398 snprintf(m_err, ERRSTR_MAXSZ, 399 "rela has type %lx but expected %lx\n", 400 (long)rela_info(rel), rela_type); 401 return -1; 402 } 403 404 if (long_size == 4) 405 *(uint32_t *)ptr = rela_addend(rel); 406 else 407 *(uint64_t *)ptr = rela_addend(rel); 408 ptr += long_size; 409 count++; 410 } 411 } 412 } 413 return count; 414 } 415 416 /* Put the sorted vals back into the relocation elements */ 417 static void replace_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 418 { 419 Elf_Shdr *shdr_start; 420 Elf_Rela *rel; 421 unsigned int shnum; 422 int shentsize; 423 424 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 425 shentsize = ehdr_shentsize(ehdr); 426 427 shnum = ehdr_shnum(ehdr); 428 if (shnum == SHN_UNDEF) 429 shnum = shdr_size(shdr_start); 430 431 for (int i = 0; i < shnum; i++) { 432 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 433 void *end; 434 435 if (shdr_type(shdr) != SHT_RELA) 436 continue; 437 438 rel = (void *)ehdr + shdr_offset(shdr); 439 end = (void *)rel + shdr_size(shdr); 440 441 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 442 uint64_t offset = rela_offset(rel); 443 444 if (offset >= start_loc && offset < start_loc + size) { 445 if (long_size == 4) 446 rela_write_addend(rel, *(uint32_t *)ptr); 447 else 448 rela_write_addend(rel, *(uint64_t *)ptr); 449 ptr += long_size; 450 } 451 } 452 } 453 } 454 455 static int fill_addrs(void *ptr, uint64_t size, void *addrs) 456 { 457 void *end = ptr + size; 458 int count = 0; 459 460 for (; ptr < end; ptr += long_size, addrs += long_size, count++) { 461 if (long_size == 4) 462 *(uint32_t *)ptr = elf_parser.r(addrs); 463 else 464 *(uint64_t *)ptr = elf_parser.r8(addrs); 465 } 466 return count; 467 } 468 469 static void replace_addrs(void *ptr, uint64_t size, void *addrs) 470 { 471 void *end = ptr + size; 472 473 for (; ptr < end; ptr += long_size, addrs += long_size) { 474 if (long_size == 4) 475 elf_parser.w(*(uint32_t *)ptr, addrs); 476 else 477 elf_parser.w8(*(uint64_t *)ptr, addrs); 478 } 479 } 480 481 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 482 static void *sort_mcount_loc(void *arg) 483 { 484 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 485 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec) 486 + shdr_offset(emloc->init_data_sec); 487 uint64_t size = emloc->stop_mcount_loc - emloc->start_mcount_loc; 488 unsigned char *start_loc = (void *)emloc->ehdr + offset; 489 Elf_Ehdr *ehdr = emloc->ehdr; 490 void *e_msg = NULL; 491 void *vals; 492 int count; 493 494 vals = malloc(long_size * size); 495 if (!vals) { 496 snprintf(m_err, ERRSTR_MAXSZ, "Failed to allocate sort array"); 497 pthread_exit(m_err); 498 } 499 500 if (sort_reloc) { 501 count = fill_relocs(vals, size, ehdr, emloc->start_mcount_loc); 502 /* gcc may use relocs to save the addresses, but clang does not. */ 503 if (!count) { 504 count = fill_addrs(vals, size, start_loc); 505 sort_reloc = 0; 506 } 507 } else 508 count = fill_addrs(vals, size, start_loc); 509 510 if (count < 0) { 511 e_msg = m_err; 512 goto out; 513 } 514 515 if (count != size / long_size) { 516 snprintf(m_err, ERRSTR_MAXSZ, "Expected %u mcount elements but found %u\n", 517 (int)(size / long_size), count); 518 e_msg = m_err; 519 goto out; 520 } 521 522 /* zero out any locations not found by function list */ 523 if (function_list_size) { 524 for (void *ptr = vals; ptr < vals + size; ptr += long_size) { 525 uint64_t key; 526 527 key = long_size == 4 ? *(uint32_t *)ptr : *(uint64_t *)ptr; 528 if (!find_func(key)) { 529 if (long_size == 4) 530 *(uint32_t *)ptr = 0; 531 else 532 *(uint64_t *)ptr = 0; 533 } 534 } 535 } 536 537 compare_values = long_size == 4 ? compare_values_32 : compare_values_64; 538 539 qsort(vals, count, long_size, compare_values); 540 541 if (sort_reloc) 542 replace_relocs(vals, size, ehdr, emloc->start_mcount_loc); 543 else 544 replace_addrs(vals, size, start_loc); 545 546 out: 547 free(vals); 548 549 pthread_exit(e_msg); 550 } 551 552 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ 553 static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec, 554 const char *strtab) 555 { 556 Elf_Sym *sym, *end_sym; 557 int symentsize = shdr_entsize(symtab_sec); 558 int found = 0; 559 560 sym = (void *)emloc->ehdr + shdr_offset(symtab_sec); 561 end_sym = (void *)sym + shdr_size(symtab_sec); 562 563 while (sym < end_sym) { 564 if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) { 565 emloc->start_mcount_loc = sym_value(sym); 566 if (++found == 2) 567 break; 568 } else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) { 569 emloc->stop_mcount_loc = sym_value(sym); 570 if (++found == 2) 571 break; 572 } 573 sym = (void *)sym + symentsize; 574 } 575 576 if (!emloc->start_mcount_loc) { 577 fprintf(stderr, "get start_mcount_loc error!"); 578 return; 579 } 580 581 if (!emloc->stop_mcount_loc) { 582 fprintf(stderr, "get stop_mcount_loc error!"); 583 return; 584 } 585 } 586 #else /* MCOUNT_SORT_ENABLED */ 587 static inline int parse_symbols(const char *fname) { return 0; } 588 #endif 589 590 static int do_sort(Elf_Ehdr *ehdr, 591 char const *const fname, 592 table_sort_t custom_sort) 593 { 594 int rc = -1; 595 Elf_Shdr *shdr_start; 596 Elf_Shdr *strtab_sec = NULL; 597 Elf_Shdr *symtab_sec = NULL; 598 Elf_Shdr *extab_sec = NULL; 599 Elf_Shdr *string_sec; 600 Elf_Sym *sym; 601 const Elf_Sym *symtab; 602 Elf32_Word *symtab_shndx = NULL; 603 Elf_Sym *sort_needed_sym = NULL; 604 Elf_Shdr *sort_needed_sec; 605 uint32_t *sort_needed_loc; 606 void *sym_start; 607 void *sym_end; 608 const char *secstrings; 609 const char *strtab; 610 char *extab_image; 611 int sort_need_index; 612 int symentsize; 613 int shentsize; 614 int idx; 615 int i; 616 unsigned int shnum; 617 unsigned int shstrndx; 618 #ifdef MCOUNT_SORT_ENABLED 619 struct elf_mcount_loc mstruct = {0}; 620 #endif 621 #ifdef UNWINDER_ORC_ENABLED 622 unsigned int orc_ip_size = 0; 623 unsigned int orc_size = 0; 624 unsigned int orc_num_entries = 0; 625 #endif 626 627 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 628 shentsize = ehdr_shentsize(ehdr); 629 630 shstrndx = ehdr_shstrndx(ehdr); 631 if (shstrndx == SHN_XINDEX) 632 shstrndx = shdr_link(shdr_start); 633 string_sec = get_index(shdr_start, shentsize, shstrndx); 634 secstrings = (const char *)ehdr + shdr_offset(string_sec); 635 636 shnum = ehdr_shnum(ehdr); 637 if (shnum == SHN_UNDEF) 638 shnum = shdr_size(shdr_start); 639 640 for (i = 0; i < shnum; i++) { 641 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 642 643 idx = shdr_name(shdr); 644 if (!strcmp(secstrings + idx, "__ex_table")) 645 extab_sec = shdr; 646 if (!strcmp(secstrings + idx, ".symtab")) 647 symtab_sec = shdr; 648 if (!strcmp(secstrings + idx, ".strtab")) 649 strtab_sec = shdr; 650 651 if (shdr_type(shdr) == SHT_SYMTAB_SHNDX) 652 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 653 shdr_offset(shdr)); 654 655 #ifdef MCOUNT_SORT_ENABLED 656 /* locate the .init.data section in vmlinux */ 657 if (!strcmp(secstrings + idx, ".init.data")) 658 mstruct.init_data_sec = shdr; 659 #endif 660 661 #ifdef UNWINDER_ORC_ENABLED 662 /* locate the ORC unwind tables */ 663 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 664 orc_ip_size = shdr_size(shdr); 665 g_orc_ip_table = (int *)((void *)ehdr + 666 shdr_offset(shdr)); 667 } 668 if (!strcmp(secstrings + idx, ".orc_unwind")) { 669 orc_size = shdr_size(shdr); 670 g_orc_table = (struct orc_entry *)((void *)ehdr + 671 shdr_offset(shdr)); 672 } 673 #endif 674 } /* for loop */ 675 676 #ifdef UNWINDER_ORC_ENABLED 677 if (!g_orc_ip_table || !g_orc_table) { 678 fprintf(stderr, 679 "incomplete ORC unwind tables in file: %s\n", fname); 680 goto out; 681 } 682 683 orc_num_entries = orc_ip_size / sizeof(int); 684 if (orc_ip_size % sizeof(int) != 0 || 685 orc_size % sizeof(struct orc_entry) != 0 || 686 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 687 fprintf(stderr, 688 "inconsistent ORC unwind table entries in file: %s\n", 689 fname); 690 goto out; 691 } 692 693 /* create thread to sort ORC unwind tables concurrently */ 694 if (pthread_create(&orc_sort_thread, NULL, 695 sort_orctable, &orc_ip_size)) { 696 fprintf(stderr, 697 "pthread_create orc_sort_thread failed '%s': %s\n", 698 strerror(errno), fname); 699 goto out; 700 } 701 #endif 702 if (!extab_sec) { 703 fprintf(stderr, "no __ex_table in file: %s\n", fname); 704 goto out; 705 } 706 707 if (!symtab_sec) { 708 fprintf(stderr, "no .symtab in file: %s\n", fname); 709 goto out; 710 } 711 712 if (!strtab_sec) { 713 fprintf(stderr, "no .strtab in file: %s\n", fname); 714 goto out; 715 } 716 717 extab_image = (void *)ehdr + shdr_offset(extab_sec); 718 strtab = (const char *)ehdr + shdr_offset(strtab_sec); 719 symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec)); 720 721 #ifdef MCOUNT_SORT_ENABLED 722 mstruct.ehdr = ehdr; 723 get_mcount_loc(&mstruct, symtab_sec, strtab); 724 725 if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) { 726 fprintf(stderr, 727 "incomplete mcount's sort in file: %s\n", 728 fname); 729 goto out; 730 } 731 732 /* create thread to sort mcount_loc concurrently */ 733 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { 734 fprintf(stderr, 735 "pthread_create mcount_sort_thread failed '%s': %s\n", 736 strerror(errno), fname); 737 goto out; 738 } 739 #endif 740 741 if (custom_sort) { 742 custom_sort(extab_image, shdr_size(extab_sec)); 743 } else { 744 int num_entries = shdr_size(extab_sec) / extable_ent_size; 745 qsort(extab_image, num_entries, 746 extable_ent_size, compare_extable); 747 } 748 749 /* find the flag main_extable_sort_needed */ 750 sym_start = (void *)ehdr + shdr_offset(symtab_sec); 751 sym_end = sym_start + shdr_size(symtab_sec); 752 symentsize = shdr_entsize(symtab_sec); 753 754 for (sym = sym_start; (void *)sym + symentsize < sym_end; 755 sym = (void *)sym + symentsize) { 756 if (sym_type(sym) != STT_OBJECT) 757 continue; 758 if (!strcmp(strtab + sym_name(sym), 759 "main_extable_sort_needed")) { 760 sort_needed_sym = sym; 761 break; 762 } 763 } 764 765 if (!sort_needed_sym) { 766 fprintf(stderr, 767 "no main_extable_sort_needed symbol in file: %s\n", 768 fname); 769 goto out; 770 } 771 772 sort_need_index = get_secindex(sym_shndx(sym), 773 ((void *)sort_needed_sym - (void *)symtab) / symentsize, 774 symtab_shndx); 775 sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index); 776 sort_needed_loc = (void *)ehdr + 777 shdr_offset(sort_needed_sec) + 778 sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec); 779 780 /* extable has been sorted, clear the flag */ 781 elf_parser.w(0, sort_needed_loc); 782 rc = 0; 783 784 out: 785 #ifdef UNWINDER_ORC_ENABLED 786 if (orc_sort_thread) { 787 void *retval = NULL; 788 /* wait for ORC tables sort done */ 789 rc = pthread_join(orc_sort_thread, &retval); 790 if (rc) { 791 fprintf(stderr, 792 "pthread_join failed '%s': %s\n", 793 strerror(errno), fname); 794 } else if (retval) { 795 rc = -1; 796 fprintf(stderr, 797 "failed to sort ORC tables '%s': %s\n", 798 (char *)retval, fname); 799 } 800 } 801 #endif 802 803 #ifdef MCOUNT_SORT_ENABLED 804 if (mcount_sort_thread) { 805 void *retval = NULL; 806 /* wait for mcount sort done */ 807 rc = pthread_join(mcount_sort_thread, &retval); 808 if (rc) { 809 fprintf(stderr, 810 "pthread_join failed '%s': %s\n", 811 strerror(errno), fname); 812 } else if (retval) { 813 rc = -1; 814 fprintf(stderr, 815 "failed to sort mcount '%s': %s\n", 816 (char *)retval, fname); 817 } 818 } 819 #endif 820 return rc; 821 } 822 823 static int compare_relative_table(const void *a, const void *b) 824 { 825 int32_t av = (int32_t)elf_parser.r(a); 826 int32_t bv = (int32_t)elf_parser.r(b); 827 828 if (av < bv) 829 return -1; 830 if (av > bv) 831 return 1; 832 return 0; 833 } 834 835 static void sort_relative_table(char *extab_image, int image_size) 836 { 837 int i = 0; 838 839 /* 840 * Do the same thing the runtime sort does, first normalize to 841 * being relative to the start of the section. 842 */ 843 while (i < image_size) { 844 uint32_t *loc = (uint32_t *)(extab_image + i); 845 elf_parser.w(elf_parser.r(loc) + i, loc); 846 i += 4; 847 } 848 849 qsort(extab_image, image_size / 8, 8, compare_relative_table); 850 851 /* Now denormalize. */ 852 i = 0; 853 while (i < image_size) { 854 uint32_t *loc = (uint32_t *)(extab_image + i); 855 elf_parser.w(elf_parser.r(loc) - i, loc); 856 i += 4; 857 } 858 } 859 860 static void sort_relative_table_with_data(char *extab_image, int image_size) 861 { 862 int i = 0; 863 864 while (i < image_size) { 865 uint32_t *loc = (uint32_t *)(extab_image + i); 866 867 elf_parser.w(elf_parser.r(loc) + i, loc); 868 elf_parser.w(elf_parser.r(loc + 1) + i + 4, loc + 1); 869 /* Don't touch the fixup type or data */ 870 871 i += sizeof(uint32_t) * 3; 872 } 873 874 qsort(extab_image, image_size / 12, 12, compare_relative_table); 875 876 i = 0; 877 while (i < image_size) { 878 uint32_t *loc = (uint32_t *)(extab_image + i); 879 880 elf_parser.w(elf_parser.r(loc) - i, loc); 881 elf_parser.w(elf_parser.r(loc + 1) - (i + 4), loc + 1); 882 /* Don't touch the fixup type or data */ 883 884 i += sizeof(uint32_t) * 3; 885 } 886 } 887 888 static int do_file(char const *const fname, void *addr) 889 { 890 Elf_Ehdr *ehdr = addr; 891 table_sort_t custom_sort = NULL; 892 893 switch (elf_map_machine(ehdr)) { 894 case EM_AARCH64: 895 #ifdef MCOUNT_SORT_ENABLED 896 sort_reloc = true; 897 rela_type = 0x403; 898 /* arm64 uses patchable function entry placing before function */ 899 before_func = 8; 900 #endif 901 /* fallthrough */ 902 case EM_386: 903 case EM_LOONGARCH: 904 case EM_RISCV: 905 case EM_S390: 906 case EM_X86_64: 907 custom_sort = sort_relative_table_with_data; 908 break; 909 case EM_PARISC: 910 case EM_PPC: 911 case EM_PPC64: 912 custom_sort = sort_relative_table; 913 break; 914 case EM_ARCOMPACT: 915 case EM_ARCV2: 916 case EM_ARM: 917 case EM_MICROBLAZE: 918 case EM_MIPS: 919 case EM_XTENSA: 920 break; 921 default: 922 fprintf(stderr, "unrecognized e_machine %d %s\n", 923 elf_parser.r2(&ehdr->e32.e_machine), fname); 924 return -1; 925 } 926 927 switch (elf_map_long_size(addr)) { 928 case 4: 929 compare_extable = compare_extable_32, 930 long_size = 4; 931 extable_ent_size = 8; 932 933 if (elf_parser.r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) || 934 elf_parser.r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) { 935 fprintf(stderr, 936 "unrecognized ET_EXEC/ET_DYN file: %s\n", fname); 937 return -1; 938 } 939 940 break; 941 case 8: 942 compare_extable = compare_extable_64, 943 long_size = 8; 944 extable_ent_size = 16; 945 946 if (elf_parser.r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) || 947 elf_parser.r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) { 948 fprintf(stderr, 949 "unrecognized ET_EXEC/ET_DYN file: %s\n", 950 fname); 951 return -1; 952 } 953 954 break; 955 default: 956 fprintf(stderr, "unrecognized ELF class %d %s\n", 957 ehdr->e32.e_ident[EI_CLASS], fname); 958 return -1; 959 } 960 961 return do_sort(ehdr, fname, custom_sort); 962 } 963 964 int main(int argc, char *argv[]) 965 { 966 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */ 967 size_t size = 0; 968 void *addr = NULL; 969 int c; 970 971 while ((c = getopt(argc, argv, "s:")) >= 0) { 972 switch (c) { 973 case 's': 974 if (parse_symbols(optarg) < 0) { 975 fprintf(stderr, "Could not parse %s\n", optarg); 976 return -1; 977 } 978 break; 979 default: 980 fprintf(stderr, "usage: sorttable [-s nm-file] vmlinux...\n"); 981 return 0; 982 } 983 } 984 985 if ((argc - optind) < 1) { 986 fprintf(stderr, "usage: sorttable vmlinux...\n"); 987 return 0; 988 } 989 990 /* Process each file in turn, allowing deep failure. */ 991 for (i = optind; i < argc; i++) { 992 addr = elf_map(argv[i], &size, (1 << ET_EXEC) | (1 << ET_DYN)); 993 if (!addr) { 994 ++n_error; 995 continue; 996 } 997 998 if (do_file(argv[i], addr)) 999 ++n_error; 1000 1001 elf_unmap(addr, size); 1002 } 1003 1004 return !!n_error; 1005 } 1006