1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * sorttable.c: Sort the kernel's table 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <shile.zhang@linux.alibaba.com> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Based on code taken from recortmcount.c which is: 12 * 13 * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved. 14 * 15 * Restructured to fit Linux format, as well as other updates: 16 * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc. 17 */ 18 19 /* 20 * Strategy: alter the vmlinux file in-place. 21 */ 22 23 #include <sys/types.h> 24 #include <sys/stat.h> 25 #include <getopt.h> 26 #include <fcntl.h> 27 #include <stdio.h> 28 #include <stdlib.h> 29 #include <stdbool.h> 30 #include <string.h> 31 #include <unistd.h> 32 #include <errno.h> 33 #include <pthread.h> 34 35 #include "elf-parse.h" 36 37 #ifndef EM_ARCOMPACT 38 #define EM_ARCOMPACT 93 39 #endif 40 41 #ifndef EM_XTENSA 42 #define EM_XTENSA 94 43 #endif 44 45 #ifndef EM_AARCH64 46 #define EM_AARCH64 183 47 #endif 48 49 #ifndef EM_MICROBLAZE 50 #define EM_MICROBLAZE 189 51 #endif 52 53 #ifndef EM_ARCV2 54 #define EM_ARCV2 195 55 #endif 56 57 #ifndef EM_RISCV 58 #define EM_RISCV 243 59 #endif 60 61 #ifndef EM_LOONGARCH 62 #define EM_LOONGARCH 258 63 #endif 64 65 typedef void (*table_sort_t)(char *, int); 66 67 /* 68 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of 69 * the way to -256..-1, to avoid conflicting with real section 70 * indices. 71 */ 72 #define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1)) 73 74 static inline int is_shndx_special(unsigned int i) 75 { 76 return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE; 77 } 78 79 /* Accessor for sym->st_shndx, hides ugliness of "64k sections" */ 80 static inline unsigned int get_secindex(unsigned int shndx, 81 unsigned int sym_offs, 82 const Elf32_Word *symtab_shndx_start) 83 { 84 if (is_shndx_special(shndx)) 85 return SPECIAL(shndx); 86 if (shndx != SHN_XINDEX) 87 return shndx; 88 return elf_parser.r(&symtab_shndx_start[sym_offs]); 89 } 90 91 static int compare_extable_32(const void *a, const void *b) 92 { 93 Elf32_Addr av = elf_parser.r(a); 94 Elf32_Addr bv = elf_parser.r(b); 95 96 if (av < bv) 97 return -1; 98 return av > bv; 99 } 100 101 static int compare_extable_64(const void *a, const void *b) 102 { 103 Elf64_Addr av = elf_parser.r8(a); 104 Elf64_Addr bv = elf_parser.r8(b); 105 106 if (av < bv) 107 return -1; 108 return av > bv; 109 } 110 111 static int (*compare_extable)(const void *a, const void *b); 112 113 static inline void *get_index(void *start, int entsize, int index) 114 { 115 return start + (entsize * index); 116 } 117 118 static int extable_ent_size; 119 static int long_size __maybe_unused; 120 121 #define ERRSTR_MAXSZ 256 122 123 #ifdef UNWINDER_ORC_ENABLED 124 /* ORC unwinder only support X86_64 */ 125 #include <asm/orc_types.h> 126 127 static char g_err[ERRSTR_MAXSZ]; 128 static int *g_orc_ip_table; 129 static struct orc_entry *g_orc_table; 130 131 static pthread_t orc_sort_thread; 132 133 static inline unsigned long orc_ip(const int *ip) 134 { 135 return (unsigned long)ip + *ip; 136 } 137 138 static int orc_sort_cmp(const void *_a, const void *_b) 139 { 140 struct orc_entry *orc_a, *orc_b; 141 const int *a = g_orc_ip_table + *(int *)_a; 142 const int *b = g_orc_ip_table + *(int *)_b; 143 unsigned long a_val = orc_ip(a); 144 unsigned long b_val = orc_ip(b); 145 146 if (a_val > b_val) 147 return 1; 148 if (a_val < b_val) 149 return -1; 150 151 /* 152 * The "weak" section terminator entries need to always be on the left 153 * to ensure the lookup code skips them in favor of real entries. 154 * These terminator entries exist to handle any gaps created by 155 * whitelisted .o files which didn't get objtool generation. 156 */ 157 orc_a = g_orc_table + (a - g_orc_ip_table); 158 orc_b = g_orc_table + (b - g_orc_ip_table); 159 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED) 160 return 0; 161 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 162 } 163 164 static void *sort_orctable(void *arg) 165 { 166 int i; 167 int *idxs = NULL; 168 int *tmp_orc_ip_table = NULL; 169 struct orc_entry *tmp_orc_table = NULL; 170 unsigned int *orc_ip_size = (unsigned int *)arg; 171 unsigned int num_entries = *orc_ip_size / sizeof(int); 172 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 173 174 idxs = (int *)malloc(*orc_ip_size); 175 if (!idxs) { 176 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 177 strerror(errno)); 178 pthread_exit(g_err); 179 } 180 181 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 182 if (!tmp_orc_ip_table) { 183 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 184 strerror(errno)); 185 pthread_exit(g_err); 186 } 187 188 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 189 if (!tmp_orc_table) { 190 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 191 strerror(errno)); 192 pthread_exit(g_err); 193 } 194 195 /* initialize indices array, convert ip_table to absolute address */ 196 for (i = 0; i < num_entries; i++) { 197 idxs[i] = i; 198 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 199 } 200 memcpy(tmp_orc_table, g_orc_table, orc_size); 201 202 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 203 204 for (i = 0; i < num_entries; i++) { 205 if (idxs[i] == i) 206 continue; 207 208 /* convert back to relative address */ 209 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 210 g_orc_table[i] = tmp_orc_table[idxs[i]]; 211 } 212 213 free(idxs); 214 free(tmp_orc_ip_table); 215 free(tmp_orc_table); 216 pthread_exit(NULL); 217 } 218 #endif 219 220 #ifdef MCOUNT_SORT_ENABLED 221 222 static int compare_values_64(const void *a, const void *b) 223 { 224 uint64_t av = *(uint64_t *)a; 225 uint64_t bv = *(uint64_t *)b; 226 227 if (av < bv) 228 return -1; 229 return av > bv; 230 } 231 232 static int compare_values_32(const void *a, const void *b) 233 { 234 uint32_t av = *(uint32_t *)a; 235 uint32_t bv = *(uint32_t *)b; 236 237 if (av < bv) 238 return -1; 239 return av > bv; 240 } 241 242 static int (*compare_values)(const void *a, const void *b); 243 244 /* Only used for sorting mcount table */ 245 static void rela_write_addend(Elf_Rela *rela, uint64_t val) 246 { 247 elf_parser.rela_write_addend(rela, val); 248 } 249 250 struct func_info { 251 uint64_t addr; 252 uint64_t size; 253 }; 254 255 /* List of functions created by: nm -S vmlinux */ 256 static struct func_info *function_list; 257 static int function_list_size; 258 259 /* Allocate functions in 1k blocks */ 260 #define FUNC_BLK_SIZE 1024 261 #define FUNC_BLK_MASK (FUNC_BLK_SIZE - 1) 262 263 static int add_field(uint64_t addr, uint64_t size) 264 { 265 struct func_info *fi; 266 int fsize = function_list_size; 267 268 if (!(fsize & FUNC_BLK_MASK)) { 269 fsize += FUNC_BLK_SIZE; 270 fi = realloc(function_list, fsize * sizeof(struct func_info)); 271 if (!fi) 272 return -1; 273 function_list = fi; 274 } 275 fi = &function_list[function_list_size++]; 276 fi->addr = addr; 277 fi->size = size; 278 return 0; 279 } 280 281 /* Used for when mcount/fentry is before the function entry */ 282 static int before_func; 283 284 /* Only return match if the address lies inside the function size */ 285 static int cmp_func_addr(const void *K, const void *A) 286 { 287 uint64_t key = *(const uint64_t *)K; 288 const struct func_info *a = A; 289 290 if (key + before_func < a->addr) 291 return -1; 292 return key >= a->addr + a->size; 293 } 294 295 /* Find the function in function list that is bounded by the function size */ 296 static int find_func(uint64_t key) 297 { 298 return bsearch(&key, function_list, function_list_size, 299 sizeof(struct func_info), cmp_func_addr) != NULL; 300 } 301 302 static int cmp_funcs(const void *A, const void *B) 303 { 304 const struct func_info *a = A; 305 const struct func_info *b = B; 306 307 if (a->addr < b->addr) 308 return -1; 309 return a->addr > b->addr; 310 } 311 312 static int parse_symbols(const char *fname) 313 { 314 FILE *fp; 315 char addr_str[20]; /* Only need 17, but round up to next int size */ 316 char size_str[20]; 317 char type; 318 319 fp = fopen(fname, "r"); 320 if (!fp) { 321 perror(fname); 322 return -1; 323 } 324 325 while (fscanf(fp, "%16s %16s %c%*[^\n]", addr_str, size_str, &type) == 3) { 326 uint64_t addr; 327 uint64_t size; 328 329 /* 330 * nm outputs size-less entries with a missing 2nd value, which 331 * means fscanf() just misread its fields. 332 * 333 * These don't matter as a call site cannot be in an empty 334 * function, so just skip them. 335 * 336 * nm pads address and size to the same width, so if their 337 * widths differ, this is a misread. 338 */ 339 if (strlen(size_str) != strlen(addr_str)) 340 continue; 341 342 /* Only care about functions */ 343 if (type != 't' && type != 'T' && type != 'W') 344 continue; 345 346 addr = strtoull(addr_str, NULL, 16); 347 size = strtoull(size_str, NULL, 16); 348 if (add_field(addr, size) < 0) 349 return -1; 350 } 351 fclose(fp); 352 353 qsort(function_list, function_list_size, sizeof(struct func_info), cmp_funcs); 354 355 return 0; 356 } 357 358 static pthread_t mcount_sort_thread; 359 static bool sort_reloc; 360 361 static long rela_type; 362 363 static char m_err[ERRSTR_MAXSZ]; 364 365 struct elf_mcount_loc { 366 Elf_Ehdr *ehdr; 367 Elf_Shdr *init_data_sec; 368 uint64_t start_mcount_loc; 369 uint64_t stop_mcount_loc; 370 }; 371 372 /* Fill the array with the content of the relocs */ 373 static int fill_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 374 { 375 Elf_Shdr *shdr_start; 376 Elf_Rela *rel; 377 unsigned int shnum; 378 unsigned int count = 0; 379 int shentsize; 380 void *array_end = ptr + size; 381 382 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 383 shentsize = ehdr_shentsize(ehdr); 384 385 shnum = ehdr_shnum(ehdr); 386 if (shnum == SHN_UNDEF) 387 shnum = shdr_size(shdr_start); 388 389 for (int i = 0; i < shnum; i++) { 390 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 391 void *end; 392 393 if (shdr_type(shdr) != SHT_RELA) 394 continue; 395 396 rel = (void *)ehdr + shdr_offset(shdr); 397 end = (void *)rel + shdr_size(shdr); 398 399 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 400 uint64_t offset = rela_offset(rel); 401 402 if (offset >= start_loc && offset < start_loc + size) { 403 if (ptr + long_size > array_end) { 404 snprintf(m_err, ERRSTR_MAXSZ, 405 "Too many relocations"); 406 return -1; 407 } 408 409 /* Make sure this has the correct type */ 410 if (rela_info(rel) != rela_type) { 411 snprintf(m_err, ERRSTR_MAXSZ, 412 "rela has type %lx but expected %lx\n", 413 (long)rela_info(rel), rela_type); 414 return -1; 415 } 416 417 if (long_size == 4) 418 *(uint32_t *)ptr = rela_addend(rel); 419 else 420 *(uint64_t *)ptr = rela_addend(rel); 421 ptr += long_size; 422 count++; 423 } 424 } 425 } 426 return count; 427 } 428 429 /* Put the sorted vals back into the relocation elements */ 430 static void replace_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc) 431 { 432 Elf_Shdr *shdr_start; 433 Elf_Rela *rel; 434 unsigned int shnum; 435 int shentsize; 436 437 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 438 shentsize = ehdr_shentsize(ehdr); 439 440 shnum = ehdr_shnum(ehdr); 441 if (shnum == SHN_UNDEF) 442 shnum = shdr_size(shdr_start); 443 444 for (int i = 0; i < shnum; i++) { 445 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 446 void *end; 447 448 if (shdr_type(shdr) != SHT_RELA) 449 continue; 450 451 rel = (void *)ehdr + shdr_offset(shdr); 452 end = (void *)rel + shdr_size(shdr); 453 454 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) { 455 uint64_t offset = rela_offset(rel); 456 457 if (offset >= start_loc && offset < start_loc + size) { 458 if (long_size == 4) 459 rela_write_addend(rel, *(uint32_t *)ptr); 460 else 461 rela_write_addend(rel, *(uint64_t *)ptr); 462 ptr += long_size; 463 } 464 } 465 } 466 } 467 468 static int fill_addrs(void *ptr, uint64_t size, void *addrs) 469 { 470 void *end = ptr + size; 471 int count = 0; 472 473 for (; ptr < end; ptr += long_size, addrs += long_size, count++) { 474 if (long_size == 4) 475 *(uint32_t *)ptr = elf_parser.r(addrs); 476 else 477 *(uint64_t *)ptr = elf_parser.r8(addrs); 478 } 479 return count; 480 } 481 482 static void replace_addrs(void *ptr, uint64_t size, void *addrs) 483 { 484 void *end = ptr + size; 485 486 for (; ptr < end; ptr += long_size, addrs += long_size) { 487 if (long_size == 4) 488 elf_parser.w(*(uint32_t *)ptr, addrs); 489 else 490 elf_parser.w8(*(uint64_t *)ptr, addrs); 491 } 492 } 493 494 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 495 static void *sort_mcount_loc(void *arg) 496 { 497 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 498 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec) 499 + shdr_offset(emloc->init_data_sec); 500 uint64_t size = emloc->stop_mcount_loc - emloc->start_mcount_loc; 501 unsigned char *start_loc = (void *)emloc->ehdr + offset; 502 Elf_Ehdr *ehdr = emloc->ehdr; 503 void *e_msg = NULL; 504 void *vals; 505 int count; 506 507 vals = malloc(long_size * size); 508 if (!vals) { 509 snprintf(m_err, ERRSTR_MAXSZ, "Failed to allocate sort array"); 510 pthread_exit(m_err); 511 } 512 513 if (sort_reloc) { 514 count = fill_relocs(vals, size, ehdr, emloc->start_mcount_loc); 515 /* gcc may use relocs to save the addresses, but clang does not. */ 516 if (!count) { 517 count = fill_addrs(vals, size, start_loc); 518 sort_reloc = 0; 519 } 520 } else 521 count = fill_addrs(vals, size, start_loc); 522 523 if (count < 0) { 524 e_msg = m_err; 525 goto out; 526 } 527 528 if (count != size / long_size) { 529 snprintf(m_err, ERRSTR_MAXSZ, "Expected %u mcount elements but found %u\n", 530 (int)(size / long_size), count); 531 e_msg = m_err; 532 goto out; 533 } 534 535 /* zero out any locations not found by function list */ 536 if (function_list_size) { 537 for (void *ptr = vals; ptr < vals + size; ptr += long_size) { 538 uint64_t key; 539 540 key = long_size == 4 ? *(uint32_t *)ptr : *(uint64_t *)ptr; 541 if (!find_func(key)) { 542 if (long_size == 4) 543 *(uint32_t *)ptr = 0; 544 else 545 *(uint64_t *)ptr = 0; 546 } 547 } 548 } 549 550 compare_values = long_size == 4 ? compare_values_32 : compare_values_64; 551 552 qsort(vals, count, long_size, compare_values); 553 554 if (sort_reloc) 555 replace_relocs(vals, size, ehdr, emloc->start_mcount_loc); 556 else 557 replace_addrs(vals, size, start_loc); 558 559 out: 560 free(vals); 561 562 pthread_exit(e_msg); 563 } 564 565 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ 566 static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec, 567 const char *strtab) 568 { 569 Elf_Sym *sym, *end_sym; 570 int symentsize = shdr_entsize(symtab_sec); 571 int found = 0; 572 573 sym = (void *)emloc->ehdr + shdr_offset(symtab_sec); 574 end_sym = (void *)sym + shdr_size(symtab_sec); 575 576 while (sym < end_sym) { 577 if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) { 578 emloc->start_mcount_loc = sym_value(sym); 579 if (++found == 2) 580 break; 581 } else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) { 582 emloc->stop_mcount_loc = sym_value(sym); 583 if (++found == 2) 584 break; 585 } 586 sym = (void *)sym + symentsize; 587 } 588 589 if (!emloc->start_mcount_loc) { 590 fprintf(stderr, "get start_mcount_loc error!"); 591 return; 592 } 593 594 if (!emloc->stop_mcount_loc) { 595 fprintf(stderr, "get stop_mcount_loc error!"); 596 return; 597 } 598 } 599 #else /* MCOUNT_SORT_ENABLED */ 600 static inline int parse_symbols(const char *fname) { return 0; } 601 #endif 602 603 static int do_sort(Elf_Ehdr *ehdr, 604 char const *const fname, 605 table_sort_t custom_sort) 606 { 607 int rc = -1; 608 Elf_Shdr *shdr_start; 609 Elf_Shdr *strtab_sec = NULL; 610 Elf_Shdr *symtab_sec = NULL; 611 Elf_Shdr *extab_sec = NULL; 612 Elf_Shdr *string_sec; 613 Elf_Sym *sym; 614 const Elf_Sym *symtab; 615 Elf32_Word *symtab_shndx = NULL; 616 Elf_Sym *sort_needed_sym = NULL; 617 Elf_Shdr *sort_needed_sec; 618 uint32_t *sort_needed_loc; 619 void *sym_start; 620 void *sym_end; 621 const char *secstrings; 622 const char *strtab; 623 char *extab_image; 624 int sort_need_index; 625 int symentsize; 626 int shentsize; 627 int idx; 628 int i; 629 unsigned int shnum; 630 unsigned int shstrndx; 631 #ifdef MCOUNT_SORT_ENABLED 632 struct elf_mcount_loc mstruct = {0}; 633 #endif 634 #ifdef UNWINDER_ORC_ENABLED 635 unsigned int orc_ip_size = 0; 636 unsigned int orc_size = 0; 637 unsigned int orc_num_entries = 0; 638 #endif 639 640 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 641 shentsize = ehdr_shentsize(ehdr); 642 643 shstrndx = ehdr_shstrndx(ehdr); 644 if (shstrndx == SHN_XINDEX) 645 shstrndx = shdr_link(shdr_start); 646 string_sec = get_index(shdr_start, shentsize, shstrndx); 647 secstrings = (const char *)ehdr + shdr_offset(string_sec); 648 649 shnum = ehdr_shnum(ehdr); 650 if (shnum == SHN_UNDEF) 651 shnum = shdr_size(shdr_start); 652 653 for (i = 0; i < shnum; i++) { 654 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 655 656 idx = shdr_name(shdr); 657 if (!strcmp(secstrings + idx, "__ex_table")) 658 extab_sec = shdr; 659 if (!strcmp(secstrings + idx, ".symtab")) 660 symtab_sec = shdr; 661 if (!strcmp(secstrings + idx, ".strtab")) 662 strtab_sec = shdr; 663 664 if (shdr_type(shdr) == SHT_SYMTAB_SHNDX) 665 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 666 shdr_offset(shdr)); 667 668 #ifdef MCOUNT_SORT_ENABLED 669 /* locate the .init.data section in vmlinux */ 670 if (!strcmp(secstrings + idx, ".init.data")) 671 mstruct.init_data_sec = shdr; 672 #endif 673 674 #ifdef UNWINDER_ORC_ENABLED 675 /* locate the ORC unwind tables */ 676 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 677 orc_ip_size = shdr_size(shdr); 678 g_orc_ip_table = (int *)((void *)ehdr + 679 shdr_offset(shdr)); 680 } 681 if (!strcmp(secstrings + idx, ".orc_unwind")) { 682 orc_size = shdr_size(shdr); 683 g_orc_table = (struct orc_entry *)((void *)ehdr + 684 shdr_offset(shdr)); 685 } 686 #endif 687 } /* for loop */ 688 689 #ifdef UNWINDER_ORC_ENABLED 690 if (!g_orc_ip_table || !g_orc_table) { 691 fprintf(stderr, 692 "incomplete ORC unwind tables in file: %s\n", fname); 693 goto out; 694 } 695 696 orc_num_entries = orc_ip_size / sizeof(int); 697 if (orc_ip_size % sizeof(int) != 0 || 698 orc_size % sizeof(struct orc_entry) != 0 || 699 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 700 fprintf(stderr, 701 "inconsistent ORC unwind table entries in file: %s\n", 702 fname); 703 goto out; 704 } 705 706 /* create thread to sort ORC unwind tables concurrently */ 707 if (pthread_create(&orc_sort_thread, NULL, 708 sort_orctable, &orc_ip_size)) { 709 fprintf(stderr, 710 "pthread_create orc_sort_thread failed '%s': %s\n", 711 strerror(errno), fname); 712 goto out; 713 } 714 #endif 715 if (!extab_sec) { 716 fprintf(stderr, "no __ex_table in file: %s\n", fname); 717 goto out; 718 } 719 720 if (!symtab_sec) { 721 fprintf(stderr, "no .symtab in file: %s\n", fname); 722 goto out; 723 } 724 725 if (!strtab_sec) { 726 fprintf(stderr, "no .strtab in file: %s\n", fname); 727 goto out; 728 } 729 730 extab_image = (void *)ehdr + shdr_offset(extab_sec); 731 strtab = (const char *)ehdr + shdr_offset(strtab_sec); 732 symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec)); 733 734 #ifdef MCOUNT_SORT_ENABLED 735 mstruct.ehdr = ehdr; 736 get_mcount_loc(&mstruct, symtab_sec, strtab); 737 738 if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) { 739 fprintf(stderr, 740 "incomplete mcount's sort in file: %s\n", 741 fname); 742 goto out; 743 } 744 745 /* create thread to sort mcount_loc concurrently */ 746 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { 747 fprintf(stderr, 748 "pthread_create mcount_sort_thread failed '%s': %s\n", 749 strerror(errno), fname); 750 goto out; 751 } 752 #endif 753 754 if (custom_sort) { 755 custom_sort(extab_image, shdr_size(extab_sec)); 756 } else { 757 int num_entries = shdr_size(extab_sec) / extable_ent_size; 758 qsort(extab_image, num_entries, 759 extable_ent_size, compare_extable); 760 } 761 762 /* find the flag main_extable_sort_needed */ 763 sym_start = (void *)ehdr + shdr_offset(symtab_sec); 764 sym_end = sym_start + shdr_size(symtab_sec); 765 symentsize = shdr_entsize(symtab_sec); 766 767 for (sym = sym_start; (void *)sym + symentsize < sym_end; 768 sym = (void *)sym + symentsize) { 769 if (sym_type(sym) != STT_OBJECT) 770 continue; 771 if (!strcmp(strtab + sym_name(sym), 772 "main_extable_sort_needed")) { 773 sort_needed_sym = sym; 774 break; 775 } 776 } 777 778 if (!sort_needed_sym) { 779 fprintf(stderr, 780 "no main_extable_sort_needed symbol in file: %s\n", 781 fname); 782 goto out; 783 } 784 785 sort_need_index = get_secindex(sym_shndx(sym), 786 ((void *)sort_needed_sym - (void *)symtab) / symentsize, 787 symtab_shndx); 788 sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index); 789 sort_needed_loc = (void *)ehdr + 790 shdr_offset(sort_needed_sec) + 791 sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec); 792 793 /* extable has been sorted, clear the flag */ 794 elf_parser.w(0, sort_needed_loc); 795 rc = 0; 796 797 out: 798 #ifdef UNWINDER_ORC_ENABLED 799 if (orc_sort_thread) { 800 void *retval = NULL; 801 /* wait for ORC tables sort done */ 802 rc = pthread_join(orc_sort_thread, &retval); 803 if (rc) { 804 fprintf(stderr, 805 "pthread_join failed '%s': %s\n", 806 strerror(errno), fname); 807 } else if (retval) { 808 rc = -1; 809 fprintf(stderr, 810 "failed to sort ORC tables '%s': %s\n", 811 (char *)retval, fname); 812 } 813 } 814 #endif 815 816 #ifdef MCOUNT_SORT_ENABLED 817 if (mcount_sort_thread) { 818 void *retval = NULL; 819 /* wait for mcount sort done */ 820 rc = pthread_join(mcount_sort_thread, &retval); 821 if (rc) { 822 fprintf(stderr, 823 "pthread_join failed '%s': %s\n", 824 strerror(errno), fname); 825 } else if (retval) { 826 rc = -1; 827 fprintf(stderr, 828 "failed to sort mcount '%s': %s\n", 829 (char *)retval, fname); 830 } 831 } 832 #endif 833 return rc; 834 } 835 836 static int compare_relative_table(const void *a, const void *b) 837 { 838 int32_t av = (int32_t)elf_parser.r(a); 839 int32_t bv = (int32_t)elf_parser.r(b); 840 841 if (av < bv) 842 return -1; 843 if (av > bv) 844 return 1; 845 return 0; 846 } 847 848 static void sort_relative_table(char *extab_image, int image_size) 849 { 850 int i = 0; 851 852 /* 853 * Do the same thing the runtime sort does, first normalize to 854 * being relative to the start of the section. 855 */ 856 while (i < image_size) { 857 uint32_t *loc = (uint32_t *)(extab_image + i); 858 elf_parser.w(elf_parser.r(loc) + i, loc); 859 i += 4; 860 } 861 862 qsort(extab_image, image_size / 8, 8, compare_relative_table); 863 864 /* Now denormalize. */ 865 i = 0; 866 while (i < image_size) { 867 uint32_t *loc = (uint32_t *)(extab_image + i); 868 elf_parser.w(elf_parser.r(loc) - i, loc); 869 i += 4; 870 } 871 } 872 873 static void sort_relative_table_with_data(char *extab_image, int image_size) 874 { 875 int i = 0; 876 877 while (i < image_size) { 878 uint32_t *loc = (uint32_t *)(extab_image + i); 879 880 elf_parser.w(elf_parser.r(loc) + i, loc); 881 elf_parser.w(elf_parser.r(loc + 1) + i + 4, loc + 1); 882 /* Don't touch the fixup type or data */ 883 884 i += sizeof(uint32_t) * 3; 885 } 886 887 qsort(extab_image, image_size / 12, 12, compare_relative_table); 888 889 i = 0; 890 while (i < image_size) { 891 uint32_t *loc = (uint32_t *)(extab_image + i); 892 893 elf_parser.w(elf_parser.r(loc) - i, loc); 894 elf_parser.w(elf_parser.r(loc + 1) - (i + 4), loc + 1); 895 /* Don't touch the fixup type or data */ 896 897 i += sizeof(uint32_t) * 3; 898 } 899 } 900 901 static int do_file(char const *const fname, void *addr) 902 { 903 Elf_Ehdr *ehdr = addr; 904 table_sort_t custom_sort = NULL; 905 906 switch (elf_map_machine(ehdr)) { 907 #ifdef MCOUNT_SORT_ENABLED 908 case EM_AARCH64: 909 /* arm64 also needs RELA-based weak-function fixups. */ 910 sort_reloc = true; 911 rela_type = 0x403; 912 /* fallthrough */ 913 case EM_RISCV: 914 /* arm64 and RISC-V place patchable entries before the function. */ 915 before_func = 8; 916 #else 917 case EM_AARCH64: 918 case EM_RISCV: 919 #endif 920 /* fallthrough */ 921 case EM_386: 922 case EM_LOONGARCH: 923 case EM_S390: 924 case EM_X86_64: 925 custom_sort = sort_relative_table_with_data; 926 break; 927 case EM_PARISC: 928 case EM_PPC: 929 case EM_PPC64: 930 custom_sort = sort_relative_table; 931 break; 932 case EM_ARCOMPACT: 933 case EM_ARCV2: 934 case EM_ARM: 935 case EM_MICROBLAZE: 936 case EM_MIPS: 937 case EM_XTENSA: 938 break; 939 default: 940 fprintf(stderr, "unrecognized e_machine %d %s\n", 941 elf_parser.r2(&ehdr->e32.e_machine), fname); 942 return -1; 943 } 944 945 switch (elf_map_long_size(addr)) { 946 case 4: 947 compare_extable = compare_extable_32, 948 long_size = 4; 949 extable_ent_size = 8; 950 951 if (elf_parser.r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) || 952 elf_parser.r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) { 953 fprintf(stderr, 954 "unrecognized ET_EXEC/ET_DYN file: %s\n", fname); 955 return -1; 956 } 957 958 break; 959 case 8: 960 compare_extable = compare_extable_64, 961 long_size = 8; 962 extable_ent_size = 16; 963 964 if (elf_parser.r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) || 965 elf_parser.r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) { 966 fprintf(stderr, 967 "unrecognized ET_EXEC/ET_DYN file: %s\n", 968 fname); 969 return -1; 970 } 971 972 break; 973 default: 974 fprintf(stderr, "unrecognized ELF class %d %s\n", 975 ehdr->e32.e_ident[EI_CLASS], fname); 976 return -1; 977 } 978 979 return do_sort(ehdr, fname, custom_sort); 980 } 981 982 int main(int argc, char *argv[]) 983 { 984 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */ 985 size_t size = 0; 986 void *addr = NULL; 987 int c; 988 989 while ((c = getopt(argc, argv, "s:")) >= 0) { 990 switch (c) { 991 case 's': 992 if (parse_symbols(optarg) < 0) { 993 fprintf(stderr, "Could not parse %s\n", optarg); 994 return -1; 995 } 996 break; 997 default: 998 fprintf(stderr, "usage: sorttable [-s nm-file] vmlinux...\n"); 999 return 0; 1000 } 1001 } 1002 1003 if ((argc - optind) < 1) { 1004 fprintf(stderr, "usage: sorttable vmlinux...\n"); 1005 return 0; 1006 } 1007 1008 /* Process each file in turn, allowing deep failure. */ 1009 for (i = optind; i < argc; i++) { 1010 addr = elf_map(argv[i], &size, (1 << ET_EXEC) | (1 << ET_DYN)); 1011 if (!addr) { 1012 ++n_error; 1013 continue; 1014 } 1015 1016 if (do_file(argv[i], addr)) 1017 ++n_error; 1018 1019 elf_unmap(addr, size); 1020 } 1021 1022 return !!n_error; 1023 } 1024