1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * sorttable.c: Sort the kernel's table 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <shile.zhang@linux.alibaba.com> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Based on code taken from recortmcount.c which is: 12 * 13 * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved. 14 * 15 * Restructured to fit Linux format, as well as other updates: 16 * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc. 17 */ 18 19 /* 20 * Strategy: alter the vmlinux file in-place. 21 */ 22 23 #include <sys/types.h> 24 #include <sys/mman.h> 25 #include <sys/stat.h> 26 #include <getopt.h> 27 #include <elf.h> 28 #include <fcntl.h> 29 #include <stdio.h> 30 #include <stdlib.h> 31 #include <string.h> 32 #include <unistd.h> 33 #include <errno.h> 34 #include <pthread.h> 35 36 #include <tools/be_byteshift.h> 37 #include <tools/le_byteshift.h> 38 39 #ifndef EM_ARCOMPACT 40 #define EM_ARCOMPACT 93 41 #endif 42 43 #ifndef EM_XTENSA 44 #define EM_XTENSA 94 45 #endif 46 47 #ifndef EM_AARCH64 48 #define EM_AARCH64 183 49 #endif 50 51 #ifndef EM_MICROBLAZE 52 #define EM_MICROBLAZE 189 53 #endif 54 55 #ifndef EM_ARCV2 56 #define EM_ARCV2 195 57 #endif 58 59 #ifndef EM_RISCV 60 #define EM_RISCV 243 61 #endif 62 63 #ifndef EM_LOONGARCH 64 #define EM_LOONGARCH 258 65 #endif 66 67 typedef union { 68 Elf32_Ehdr e32; 69 Elf64_Ehdr e64; 70 } Elf_Ehdr; 71 72 typedef union { 73 Elf32_Shdr e32; 74 Elf64_Shdr e64; 75 } Elf_Shdr; 76 77 typedef union { 78 Elf32_Sym e32; 79 Elf64_Sym e64; 80 } Elf_Sym; 81 82 static uint32_t (*r)(const uint32_t *); 83 static uint16_t (*r2)(const uint16_t *); 84 static uint64_t (*r8)(const uint64_t *); 85 static void (*w)(uint32_t, uint32_t *); 86 typedef void (*table_sort_t)(char *, int); 87 88 static struct elf_funcs { 89 int (*compare_extable)(const void *a, const void *b); 90 uint64_t (*ehdr_shoff)(Elf_Ehdr *ehdr); 91 uint16_t (*ehdr_shstrndx)(Elf_Ehdr *ehdr); 92 uint16_t (*ehdr_shentsize)(Elf_Ehdr *ehdr); 93 uint16_t (*ehdr_shnum)(Elf_Ehdr *ehdr); 94 uint64_t (*shdr_addr)(Elf_Shdr *shdr); 95 uint64_t (*shdr_offset)(Elf_Shdr *shdr); 96 uint64_t (*shdr_size)(Elf_Shdr *shdr); 97 uint64_t (*shdr_entsize)(Elf_Shdr *shdr); 98 uint32_t (*shdr_link)(Elf_Shdr *shdr); 99 uint32_t (*shdr_name)(Elf_Shdr *shdr); 100 uint32_t (*shdr_type)(Elf_Shdr *shdr); 101 uint8_t (*sym_type)(Elf_Sym *sym); 102 uint32_t (*sym_name)(Elf_Sym *sym); 103 uint64_t (*sym_value)(Elf_Sym *sym); 104 uint16_t (*sym_shndx)(Elf_Sym *sym); 105 } e; 106 107 static uint64_t ehdr64_shoff(Elf_Ehdr *ehdr) 108 { 109 return r8(&ehdr->e64.e_shoff); 110 } 111 112 static uint64_t ehdr32_shoff(Elf_Ehdr *ehdr) 113 { 114 return r(&ehdr->e32.e_shoff); 115 } 116 117 static uint64_t ehdr_shoff(Elf_Ehdr *ehdr) 118 { 119 return e.ehdr_shoff(ehdr); 120 } 121 122 #define EHDR_HALF(fn_name) \ 123 static uint16_t ehdr64_##fn_name(Elf_Ehdr *ehdr) \ 124 { \ 125 return r2(&ehdr->e64.e_##fn_name); \ 126 } \ 127 \ 128 static uint16_t ehdr32_##fn_name(Elf_Ehdr *ehdr) \ 129 { \ 130 return r2(&ehdr->e32.e_##fn_name); \ 131 } \ 132 \ 133 static uint16_t ehdr_##fn_name(Elf_Ehdr *ehdr) \ 134 { \ 135 return e.ehdr_##fn_name(ehdr); \ 136 } 137 138 EHDR_HALF(shentsize) 139 EHDR_HALF(shstrndx) 140 EHDR_HALF(shnum) 141 142 #define SHDR_WORD(fn_name) \ 143 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 144 { \ 145 return r(&shdr->e64.sh_##fn_name); \ 146 } \ 147 \ 148 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 149 { \ 150 return r(&shdr->e32.sh_##fn_name); \ 151 } \ 152 \ 153 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \ 154 { \ 155 return e.shdr_##fn_name(shdr); \ 156 } 157 158 #define SHDR_ADDR(fn_name) \ 159 static uint64_t shdr64_##fn_name(Elf_Shdr *shdr) \ 160 { \ 161 return r8(&shdr->e64.sh_##fn_name); \ 162 } \ 163 \ 164 static uint64_t shdr32_##fn_name(Elf_Shdr *shdr) \ 165 { \ 166 return r(&shdr->e32.sh_##fn_name); \ 167 } \ 168 \ 169 static uint64_t shdr_##fn_name(Elf_Shdr *shdr) \ 170 { \ 171 return e.shdr_##fn_name(shdr); \ 172 } 173 174 #define SHDR_WORD(fn_name) \ 175 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr) \ 176 { \ 177 return r(&shdr->e64.sh_##fn_name); \ 178 } \ 179 \ 180 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr) \ 181 { \ 182 return r(&shdr->e32.sh_##fn_name); \ 183 } \ 184 static uint32_t shdr_##fn_name(Elf_Shdr *shdr) \ 185 { \ 186 return e.shdr_##fn_name(shdr); \ 187 } 188 189 SHDR_ADDR(addr) 190 SHDR_ADDR(offset) 191 SHDR_ADDR(size) 192 SHDR_ADDR(entsize) 193 194 SHDR_WORD(link) 195 SHDR_WORD(name) 196 SHDR_WORD(type) 197 198 #define SYM_ADDR(fn_name) \ 199 static uint64_t sym64_##fn_name(Elf_Sym *sym) \ 200 { \ 201 return r8(&sym->e64.st_##fn_name); \ 202 } \ 203 \ 204 static uint64_t sym32_##fn_name(Elf_Sym *sym) \ 205 { \ 206 return r(&sym->e32.st_##fn_name); \ 207 } \ 208 \ 209 static uint64_t sym_##fn_name(Elf_Sym *sym) \ 210 { \ 211 return e.sym_##fn_name(sym); \ 212 } 213 214 #define SYM_WORD(fn_name) \ 215 static uint32_t sym64_##fn_name(Elf_Sym *sym) \ 216 { \ 217 return r(&sym->e64.st_##fn_name); \ 218 } \ 219 \ 220 static uint32_t sym32_##fn_name(Elf_Sym *sym) \ 221 { \ 222 return r(&sym->e32.st_##fn_name); \ 223 } \ 224 \ 225 static uint32_t sym_##fn_name(Elf_Sym *sym) \ 226 { \ 227 return e.sym_##fn_name(sym); \ 228 } 229 230 #define SYM_HALF(fn_name) \ 231 static uint16_t sym64_##fn_name(Elf_Sym *sym) \ 232 { \ 233 return r2(&sym->e64.st_##fn_name); \ 234 } \ 235 \ 236 static uint16_t sym32_##fn_name(Elf_Sym *sym) \ 237 { \ 238 return r2(&sym->e32.st_##fn_name); \ 239 } \ 240 \ 241 static uint16_t sym_##fn_name(Elf_Sym *sym) \ 242 { \ 243 return e.sym_##fn_name(sym); \ 244 } 245 246 static uint8_t sym64_type(Elf_Sym *sym) 247 { 248 return ELF64_ST_TYPE(sym->e64.st_info); 249 } 250 251 static uint8_t sym32_type(Elf_Sym *sym) 252 { 253 return ELF32_ST_TYPE(sym->e32.st_info); 254 } 255 256 static uint8_t sym_type(Elf_Sym *sym) 257 { 258 return e.sym_type(sym); 259 } 260 261 SYM_ADDR(value) 262 SYM_WORD(name) 263 SYM_HALF(shndx) 264 265 /* 266 * Get the whole file as a programming convenience in order to avoid 267 * malloc+lseek+read+free of many pieces. If successful, then mmap 268 * avoids copying unused pieces; else just read the whole file. 269 * Open for both read and write. 270 */ 271 static void *mmap_file(char const *fname, size_t *size) 272 { 273 int fd; 274 struct stat sb; 275 void *addr = NULL; 276 277 fd = open(fname, O_RDWR); 278 if (fd < 0) { 279 perror(fname); 280 return NULL; 281 } 282 if (fstat(fd, &sb) < 0) { 283 perror(fname); 284 goto out; 285 } 286 if (!S_ISREG(sb.st_mode)) { 287 fprintf(stderr, "not a regular file: %s\n", fname); 288 goto out; 289 } 290 291 addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 292 if (addr == MAP_FAILED) { 293 fprintf(stderr, "Could not mmap file: %s\n", fname); 294 goto out; 295 } 296 297 *size = sb.st_size; 298 299 out: 300 close(fd); 301 return addr; 302 } 303 304 static uint32_t rbe(const uint32_t *x) 305 { 306 return get_unaligned_be32(x); 307 } 308 309 static uint16_t r2be(const uint16_t *x) 310 { 311 return get_unaligned_be16(x); 312 } 313 314 static uint64_t r8be(const uint64_t *x) 315 { 316 return get_unaligned_be64(x); 317 } 318 319 static uint32_t rle(const uint32_t *x) 320 { 321 return get_unaligned_le32(x); 322 } 323 324 static uint16_t r2le(const uint16_t *x) 325 { 326 return get_unaligned_le16(x); 327 } 328 329 static uint64_t r8le(const uint64_t *x) 330 { 331 return get_unaligned_le64(x); 332 } 333 334 static void wbe(uint32_t val, uint32_t *x) 335 { 336 put_unaligned_be32(val, x); 337 } 338 339 static void wle(uint32_t val, uint32_t *x) 340 { 341 put_unaligned_le32(val, x); 342 } 343 344 /* 345 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of 346 * the way to -256..-1, to avoid conflicting with real section 347 * indices. 348 */ 349 #define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1)) 350 351 static inline int is_shndx_special(unsigned int i) 352 { 353 return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE; 354 } 355 356 /* Accessor for sym->st_shndx, hides ugliness of "64k sections" */ 357 static inline unsigned int get_secindex(unsigned int shndx, 358 unsigned int sym_offs, 359 const Elf32_Word *symtab_shndx_start) 360 { 361 if (is_shndx_special(shndx)) 362 return SPECIAL(shndx); 363 if (shndx != SHN_XINDEX) 364 return shndx; 365 return r(&symtab_shndx_start[sym_offs]); 366 } 367 368 static int compare_extable_32(const void *a, const void *b) 369 { 370 Elf32_Addr av = r(a); 371 Elf32_Addr bv = r(b); 372 373 if (av < bv) 374 return -1; 375 return av > bv; 376 } 377 378 static int compare_extable_64(const void *a, const void *b) 379 { 380 Elf64_Addr av = r8(a); 381 Elf64_Addr bv = r8(b); 382 383 if (av < bv) 384 return -1; 385 return av > bv; 386 } 387 388 static int compare_extable(const void *a, const void *b) 389 { 390 return e.compare_extable(a, b); 391 } 392 393 static inline void *get_index(void *start, int entsize, int index) 394 { 395 return start + (entsize * index); 396 } 397 398 static int extable_ent_size; 399 static int long_size; 400 401 402 #ifdef UNWINDER_ORC_ENABLED 403 /* ORC unwinder only support X86_64 */ 404 #include <asm/orc_types.h> 405 406 #define ERRSTR_MAXSZ 256 407 408 static char g_err[ERRSTR_MAXSZ]; 409 static int *g_orc_ip_table; 410 static struct orc_entry *g_orc_table; 411 412 static pthread_t orc_sort_thread; 413 414 static inline unsigned long orc_ip(const int *ip) 415 { 416 return (unsigned long)ip + *ip; 417 } 418 419 static int orc_sort_cmp(const void *_a, const void *_b) 420 { 421 struct orc_entry *orc_a, *orc_b; 422 const int *a = g_orc_ip_table + *(int *)_a; 423 const int *b = g_orc_ip_table + *(int *)_b; 424 unsigned long a_val = orc_ip(a); 425 unsigned long b_val = orc_ip(b); 426 427 if (a_val > b_val) 428 return 1; 429 if (a_val < b_val) 430 return -1; 431 432 /* 433 * The "weak" section terminator entries need to always be on the left 434 * to ensure the lookup code skips them in favor of real entries. 435 * These terminator entries exist to handle any gaps created by 436 * whitelisted .o files which didn't get objtool generation. 437 */ 438 orc_a = g_orc_table + (a - g_orc_ip_table); 439 orc_b = g_orc_table + (b - g_orc_ip_table); 440 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED) 441 return 0; 442 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 443 } 444 445 static void *sort_orctable(void *arg) 446 { 447 int i; 448 int *idxs = NULL; 449 int *tmp_orc_ip_table = NULL; 450 struct orc_entry *tmp_orc_table = NULL; 451 unsigned int *orc_ip_size = (unsigned int *)arg; 452 unsigned int num_entries = *orc_ip_size / sizeof(int); 453 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 454 455 idxs = (int *)malloc(*orc_ip_size); 456 if (!idxs) { 457 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 458 strerror(errno)); 459 pthread_exit(g_err); 460 } 461 462 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 463 if (!tmp_orc_ip_table) { 464 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 465 strerror(errno)); 466 pthread_exit(g_err); 467 } 468 469 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 470 if (!tmp_orc_table) { 471 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 472 strerror(errno)); 473 pthread_exit(g_err); 474 } 475 476 /* initialize indices array, convert ip_table to absolute address */ 477 for (i = 0; i < num_entries; i++) { 478 idxs[i] = i; 479 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 480 } 481 memcpy(tmp_orc_table, g_orc_table, orc_size); 482 483 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 484 485 for (i = 0; i < num_entries; i++) { 486 if (idxs[i] == i) 487 continue; 488 489 /* convert back to relative address */ 490 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 491 g_orc_table[i] = tmp_orc_table[idxs[i]]; 492 } 493 494 free(idxs); 495 free(tmp_orc_ip_table); 496 free(tmp_orc_table); 497 pthread_exit(NULL); 498 } 499 #endif 500 501 #ifdef MCOUNT_SORT_ENABLED 502 static pthread_t mcount_sort_thread; 503 504 struct elf_mcount_loc { 505 Elf_Ehdr *ehdr; 506 Elf_Shdr *init_data_sec; 507 uint64_t start_mcount_loc; 508 uint64_t stop_mcount_loc; 509 }; 510 511 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 512 static void *sort_mcount_loc(void *arg) 513 { 514 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 515 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec) 516 + shdr_offset(emloc->init_data_sec); 517 uint64_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc; 518 unsigned char *start_loc = (void *)emloc->ehdr + offset; 519 520 qsort(start_loc, count/long_size, long_size, compare_extable); 521 return NULL; 522 } 523 524 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ 525 static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec, 526 const char *strtab) 527 { 528 Elf_Sym *sym, *end_sym; 529 int symentsize = shdr_entsize(symtab_sec); 530 int found = 0; 531 532 sym = (void *)emloc->ehdr + shdr_offset(symtab_sec); 533 end_sym = (void *)sym + shdr_size(symtab_sec); 534 535 while (sym < end_sym) { 536 if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) { 537 emloc->start_mcount_loc = sym_value(sym); 538 if (++found == 2) 539 break; 540 } else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) { 541 emloc->stop_mcount_loc = sym_value(sym); 542 if (++found == 2) 543 break; 544 } 545 sym = (void *)sym + symentsize; 546 } 547 548 if (!emloc->start_mcount_loc) { 549 fprintf(stderr, "get start_mcount_loc error!"); 550 return; 551 } 552 553 if (!emloc->stop_mcount_loc) { 554 fprintf(stderr, "get stop_mcount_loc error!"); 555 return; 556 } 557 } 558 #endif 559 560 static int do_sort(Elf_Ehdr *ehdr, 561 char const *const fname, 562 table_sort_t custom_sort) 563 { 564 int rc = -1; 565 Elf_Shdr *shdr_start; 566 Elf_Shdr *strtab_sec = NULL; 567 Elf_Shdr *symtab_sec = NULL; 568 Elf_Shdr *extab_sec = NULL; 569 Elf_Shdr *string_sec; 570 Elf_Sym *sym; 571 const Elf_Sym *symtab; 572 Elf32_Word *symtab_shndx = NULL; 573 Elf_Sym *sort_needed_sym = NULL; 574 Elf_Shdr *sort_needed_sec; 575 uint32_t *sort_needed_loc; 576 void *sym_start; 577 void *sym_end; 578 const char *secstrings; 579 const char *strtab; 580 char *extab_image; 581 int sort_need_index; 582 int symentsize; 583 int shentsize; 584 int idx; 585 int i; 586 unsigned int shnum; 587 unsigned int shstrndx; 588 #ifdef MCOUNT_SORT_ENABLED 589 struct elf_mcount_loc mstruct = {0}; 590 #endif 591 #ifdef UNWINDER_ORC_ENABLED 592 unsigned int orc_ip_size = 0; 593 unsigned int orc_size = 0; 594 unsigned int orc_num_entries = 0; 595 #endif 596 597 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr)); 598 shentsize = ehdr_shentsize(ehdr); 599 600 shstrndx = ehdr_shstrndx(ehdr); 601 if (shstrndx == SHN_XINDEX) 602 shstrndx = shdr_link(shdr_start); 603 string_sec = get_index(shdr_start, shentsize, shstrndx); 604 secstrings = (const char *)ehdr + shdr_offset(string_sec); 605 606 shnum = ehdr_shnum(ehdr); 607 if (shnum == SHN_UNDEF) 608 shnum = shdr_size(shdr_start); 609 610 for (i = 0; i < shnum; i++) { 611 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i); 612 613 idx = shdr_name(shdr); 614 if (!strcmp(secstrings + idx, "__ex_table")) 615 extab_sec = shdr; 616 if (!strcmp(secstrings + idx, ".symtab")) 617 symtab_sec = shdr; 618 if (!strcmp(secstrings + idx, ".strtab")) 619 strtab_sec = shdr; 620 621 if (shdr_type(shdr) == SHT_SYMTAB_SHNDX) 622 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 623 shdr_offset(shdr)); 624 625 #ifdef MCOUNT_SORT_ENABLED 626 /* locate the .init.data section in vmlinux */ 627 if (!strcmp(secstrings + idx, ".init.data")) 628 mstruct.init_data_sec = shdr; 629 #endif 630 631 #ifdef UNWINDER_ORC_ENABLED 632 /* locate the ORC unwind tables */ 633 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 634 orc_ip_size = shdr_size(shdr); 635 g_orc_ip_table = (int *)((void *)ehdr + 636 shdr_offset(shdr)); 637 } 638 if (!strcmp(secstrings + idx, ".orc_unwind")) { 639 orc_size = shdr_size(shdr); 640 g_orc_table = (struct orc_entry *)((void *)ehdr + 641 shdr_offset(shdr)); 642 } 643 #endif 644 } /* for loop */ 645 646 #ifdef UNWINDER_ORC_ENABLED 647 if (!g_orc_ip_table || !g_orc_table) { 648 fprintf(stderr, 649 "incomplete ORC unwind tables in file: %s\n", fname); 650 goto out; 651 } 652 653 orc_num_entries = orc_ip_size / sizeof(int); 654 if (orc_ip_size % sizeof(int) != 0 || 655 orc_size % sizeof(struct orc_entry) != 0 || 656 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 657 fprintf(stderr, 658 "inconsistent ORC unwind table entries in file: %s\n", 659 fname); 660 goto out; 661 } 662 663 /* create thread to sort ORC unwind tables concurrently */ 664 if (pthread_create(&orc_sort_thread, NULL, 665 sort_orctable, &orc_ip_size)) { 666 fprintf(stderr, 667 "pthread_create orc_sort_thread failed '%s': %s\n", 668 strerror(errno), fname); 669 goto out; 670 } 671 #endif 672 if (!extab_sec) { 673 fprintf(stderr, "no __ex_table in file: %s\n", fname); 674 goto out; 675 } 676 677 if (!symtab_sec) { 678 fprintf(stderr, "no .symtab in file: %s\n", fname); 679 goto out; 680 } 681 682 if (!strtab_sec) { 683 fprintf(stderr, "no .strtab in file: %s\n", fname); 684 goto out; 685 } 686 687 extab_image = (void *)ehdr + shdr_offset(extab_sec); 688 strtab = (const char *)ehdr + shdr_offset(strtab_sec); 689 symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec)); 690 691 #ifdef MCOUNT_SORT_ENABLED 692 mstruct.ehdr = ehdr; 693 get_mcount_loc(&mstruct, symtab_sec, strtab); 694 695 if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) { 696 fprintf(stderr, 697 "incomplete mcount's sort in file: %s\n", 698 fname); 699 goto out; 700 } 701 702 /* create thread to sort mcount_loc concurrently */ 703 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { 704 fprintf(stderr, 705 "pthread_create mcount_sort_thread failed '%s': %s\n", 706 strerror(errno), fname); 707 goto out; 708 } 709 #endif 710 711 if (custom_sort) { 712 custom_sort(extab_image, shdr_size(extab_sec)); 713 } else { 714 int num_entries = shdr_size(extab_sec) / extable_ent_size; 715 qsort(extab_image, num_entries, 716 extable_ent_size, compare_extable); 717 } 718 719 /* find the flag main_extable_sort_needed */ 720 sym_start = (void *)ehdr + shdr_offset(symtab_sec); 721 sym_end = sym_start + shdr_size(symtab_sec); 722 symentsize = shdr_entsize(symtab_sec); 723 724 for (sym = sym_start; (void *)sym + symentsize < sym_end; 725 sym = (void *)sym + symentsize) { 726 if (sym_type(sym) != STT_OBJECT) 727 continue; 728 if (!strcmp(strtab + sym_name(sym), 729 "main_extable_sort_needed")) { 730 sort_needed_sym = sym; 731 break; 732 } 733 } 734 735 if (!sort_needed_sym) { 736 fprintf(stderr, 737 "no main_extable_sort_needed symbol in file: %s\n", 738 fname); 739 goto out; 740 } 741 742 sort_need_index = get_secindex(sym_shndx(sym), 743 ((void *)sort_needed_sym - (void *)symtab) / symentsize, 744 symtab_shndx); 745 sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index); 746 sort_needed_loc = (void *)ehdr + 747 shdr_offset(sort_needed_sec) + 748 sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec); 749 750 /* extable has been sorted, clear the flag */ 751 w(0, sort_needed_loc); 752 rc = 0; 753 754 out: 755 #ifdef UNWINDER_ORC_ENABLED 756 if (orc_sort_thread) { 757 void *retval = NULL; 758 /* wait for ORC tables sort done */ 759 rc = pthread_join(orc_sort_thread, &retval); 760 if (rc) { 761 fprintf(stderr, 762 "pthread_join failed '%s': %s\n", 763 strerror(errno), fname); 764 } else if (retval) { 765 rc = -1; 766 fprintf(stderr, 767 "failed to sort ORC tables '%s': %s\n", 768 (char *)retval, fname); 769 } 770 } 771 #endif 772 773 #ifdef MCOUNT_SORT_ENABLED 774 if (mcount_sort_thread) { 775 void *retval = NULL; 776 /* wait for mcount sort done */ 777 rc = pthread_join(mcount_sort_thread, &retval); 778 if (rc) { 779 fprintf(stderr, 780 "pthread_join failed '%s': %s\n", 781 strerror(errno), fname); 782 } else if (retval) { 783 rc = -1; 784 fprintf(stderr, 785 "failed to sort mcount '%s': %s\n", 786 (char *)retval, fname); 787 } 788 } 789 #endif 790 return rc; 791 } 792 793 static int compare_relative_table(const void *a, const void *b) 794 { 795 int32_t av = (int32_t)r(a); 796 int32_t bv = (int32_t)r(b); 797 798 if (av < bv) 799 return -1; 800 if (av > bv) 801 return 1; 802 return 0; 803 } 804 805 static void sort_relative_table(char *extab_image, int image_size) 806 { 807 int i = 0; 808 809 /* 810 * Do the same thing the runtime sort does, first normalize to 811 * being relative to the start of the section. 812 */ 813 while (i < image_size) { 814 uint32_t *loc = (uint32_t *)(extab_image + i); 815 w(r(loc) + i, loc); 816 i += 4; 817 } 818 819 qsort(extab_image, image_size / 8, 8, compare_relative_table); 820 821 /* Now denormalize. */ 822 i = 0; 823 while (i < image_size) { 824 uint32_t *loc = (uint32_t *)(extab_image + i); 825 w(r(loc) - i, loc); 826 i += 4; 827 } 828 } 829 830 static void sort_relative_table_with_data(char *extab_image, int image_size) 831 { 832 int i = 0; 833 834 while (i < image_size) { 835 uint32_t *loc = (uint32_t *)(extab_image + i); 836 837 w(r(loc) + i, loc); 838 w(r(loc + 1) + i + 4, loc + 1); 839 /* Don't touch the fixup type or data */ 840 841 i += sizeof(uint32_t) * 3; 842 } 843 844 qsort(extab_image, image_size / 12, 12, compare_relative_table); 845 846 i = 0; 847 while (i < image_size) { 848 uint32_t *loc = (uint32_t *)(extab_image + i); 849 850 w(r(loc) - i, loc); 851 w(r(loc + 1) - (i + 4), loc + 1); 852 /* Don't touch the fixup type or data */ 853 854 i += sizeof(uint32_t) * 3; 855 } 856 } 857 858 static int do_file(char const *const fname, void *addr) 859 { 860 Elf_Ehdr *ehdr = addr; 861 table_sort_t custom_sort = NULL; 862 863 switch (ehdr->e32.e_ident[EI_DATA]) { 864 case ELFDATA2LSB: 865 r = rle; 866 r2 = r2le; 867 r8 = r8le; 868 w = wle; 869 break; 870 case ELFDATA2MSB: 871 r = rbe; 872 r2 = r2be; 873 r8 = r8be; 874 w = wbe; 875 break; 876 default: 877 fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", 878 ehdr->e32.e_ident[EI_DATA], fname); 879 return -1; 880 } 881 882 if (memcmp(ELFMAG, ehdr->e32.e_ident, SELFMAG) != 0 || 883 (r2(&ehdr->e32.e_type) != ET_EXEC && r2(&ehdr->e32.e_type) != ET_DYN) || 884 ehdr->e32.e_ident[EI_VERSION] != EV_CURRENT) { 885 fprintf(stderr, "unrecognized ET_EXEC/ET_DYN file %s\n", fname); 886 return -1; 887 } 888 889 switch (r2(&ehdr->e32.e_machine)) { 890 case EM_386: 891 case EM_AARCH64: 892 case EM_LOONGARCH: 893 case EM_RISCV: 894 case EM_S390: 895 case EM_X86_64: 896 custom_sort = sort_relative_table_with_data; 897 break; 898 case EM_PARISC: 899 case EM_PPC: 900 case EM_PPC64: 901 custom_sort = sort_relative_table; 902 break; 903 case EM_ARCOMPACT: 904 case EM_ARCV2: 905 case EM_ARM: 906 case EM_MICROBLAZE: 907 case EM_MIPS: 908 case EM_XTENSA: 909 break; 910 default: 911 fprintf(stderr, "unrecognized e_machine %d %s\n", 912 r2(&ehdr->e32.e_machine), fname); 913 return -1; 914 } 915 916 switch (ehdr->e32.e_ident[EI_CLASS]) { 917 case ELFCLASS32: { 918 struct elf_funcs efuncs = { 919 .compare_extable = compare_extable_32, 920 .ehdr_shoff = ehdr32_shoff, 921 .ehdr_shentsize = ehdr32_shentsize, 922 .ehdr_shstrndx = ehdr32_shstrndx, 923 .ehdr_shnum = ehdr32_shnum, 924 .shdr_addr = shdr32_addr, 925 .shdr_offset = shdr32_offset, 926 .shdr_link = shdr32_link, 927 .shdr_size = shdr32_size, 928 .shdr_name = shdr32_name, 929 .shdr_type = shdr32_type, 930 .shdr_entsize = shdr32_entsize, 931 .sym_type = sym32_type, 932 .sym_name = sym32_name, 933 .sym_value = sym32_value, 934 .sym_shndx = sym32_shndx, 935 }; 936 937 e = efuncs; 938 long_size = 4; 939 extable_ent_size = 8; 940 941 if (r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) || 942 r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) { 943 fprintf(stderr, 944 "unrecognized ET_EXEC/ET_DYN file: %s\n", fname); 945 return -1; 946 } 947 948 } 949 break; 950 case ELFCLASS64: { 951 struct elf_funcs efuncs = { 952 .compare_extable = compare_extable_64, 953 .ehdr_shoff = ehdr64_shoff, 954 .ehdr_shentsize = ehdr64_shentsize, 955 .ehdr_shstrndx = ehdr64_shstrndx, 956 .ehdr_shnum = ehdr64_shnum, 957 .shdr_addr = shdr64_addr, 958 .shdr_offset = shdr64_offset, 959 .shdr_link = shdr64_link, 960 .shdr_size = shdr64_size, 961 .shdr_name = shdr64_name, 962 .shdr_type = shdr64_type, 963 .shdr_entsize = shdr64_entsize, 964 .sym_type = sym64_type, 965 .sym_name = sym64_name, 966 .sym_value = sym64_value, 967 .sym_shndx = sym64_shndx, 968 }; 969 970 e = efuncs; 971 long_size = 8; 972 extable_ent_size = 16; 973 974 if (r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) || 975 r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) { 976 fprintf(stderr, 977 "unrecognized ET_EXEC/ET_DYN file: %s\n", 978 fname); 979 return -1; 980 } 981 982 } 983 break; 984 default: 985 fprintf(stderr, "unrecognized ELF class %d %s\n", 986 ehdr->e32.e_ident[EI_CLASS], fname); 987 return -1; 988 } 989 990 return do_sort(ehdr, fname, custom_sort); 991 } 992 993 int main(int argc, char *argv[]) 994 { 995 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */ 996 size_t size = 0; 997 void *addr = NULL; 998 999 if (argc < 2) { 1000 fprintf(stderr, "usage: sorttable vmlinux...\n"); 1001 return 0; 1002 } 1003 1004 /* Process each file in turn, allowing deep failure. */ 1005 for (i = 1; i < argc; i++) { 1006 addr = mmap_file(argv[i], &size); 1007 if (!addr) { 1008 ++n_error; 1009 continue; 1010 } 1011 1012 if (do_file(argv[i], addr)) 1013 ++n_error; 1014 1015 munmap(addr, size); 1016 } 1017 1018 return !!n_error; 1019 } 1020