1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * sorttable.c: Sort the kernel's table
4 *
5 * Added ORC unwind tables sort support and other updates:
6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by:
7 * Shile Zhang <shile.zhang@linux.alibaba.com>
8 *
9 * Copyright 2011 - 2012 Cavium, Inc.
10 *
11 * Based on code taken from recortmcount.c which is:
12 *
13 * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved.
14 *
15 * Restructured to fit Linux format, as well as other updates:
16 * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
17 */
18
19 /*
20 * Strategy: alter the vmlinux file in-place.
21 */
22
23 #include <sys/types.h>
24 #include <sys/stat.h>
25 #include <getopt.h>
26 #include <fcntl.h>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <stdbool.h>
30 #include <string.h>
31 #include <unistd.h>
32 #include <errno.h>
33 #include <pthread.h>
34
35 #include "elf-parse.h"
36
37 #ifndef EM_ARCOMPACT
38 #define EM_ARCOMPACT 93
39 #endif
40
41 #ifndef EM_XTENSA
42 #define EM_XTENSA 94
43 #endif
44
45 #ifndef EM_AARCH64
46 #define EM_AARCH64 183
47 #endif
48
49 #ifndef EM_MICROBLAZE
50 #define EM_MICROBLAZE 189
51 #endif
52
53 #ifndef EM_ARCV2
54 #define EM_ARCV2 195
55 #endif
56
57 #ifndef EM_RISCV
58 #define EM_RISCV 243
59 #endif
60
61 #ifndef EM_LOONGARCH
62 #define EM_LOONGARCH 258
63 #endif
64
65 typedef void (*table_sort_t)(char *, int);
66
67 /*
68 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of
69 * the way to -256..-1, to avoid conflicting with real section
70 * indices.
71 */
72 #define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1))
73
is_shndx_special(unsigned int i)74 static inline int is_shndx_special(unsigned int i)
75 {
76 return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE;
77 }
78
79 /* Accessor for sym->st_shndx, hides ugliness of "64k sections" */
get_secindex(unsigned int shndx,unsigned int sym_offs,const Elf32_Word * symtab_shndx_start)80 static inline unsigned int get_secindex(unsigned int shndx,
81 unsigned int sym_offs,
82 const Elf32_Word *symtab_shndx_start)
83 {
84 if (is_shndx_special(shndx))
85 return SPECIAL(shndx);
86 if (shndx != SHN_XINDEX)
87 return shndx;
88 return elf_parser.r(&symtab_shndx_start[sym_offs]);
89 }
90
compare_extable_32(const void * a,const void * b)91 static int compare_extable_32(const void *a, const void *b)
92 {
93 Elf32_Addr av = elf_parser.r(a);
94 Elf32_Addr bv = elf_parser.r(b);
95
96 if (av < bv)
97 return -1;
98 return av > bv;
99 }
100
compare_extable_64(const void * a,const void * b)101 static int compare_extable_64(const void *a, const void *b)
102 {
103 Elf64_Addr av = elf_parser.r8(a);
104 Elf64_Addr bv = elf_parser.r8(b);
105
106 if (av < bv)
107 return -1;
108 return av > bv;
109 }
110
111 static int (*compare_extable)(const void *a, const void *b);
112
get_index(void * start,int entsize,int index)113 static inline void *get_index(void *start, int entsize, int index)
114 {
115 return start + (entsize * index);
116 }
117
118 static int extable_ent_size;
119 static int long_size;
120
121 #define ERRSTR_MAXSZ 256
122
123 #ifdef UNWINDER_ORC_ENABLED
124 /* ORC unwinder only support X86_64 */
125 #include <asm/orc_types.h>
126
127 static char g_err[ERRSTR_MAXSZ];
128 static int *g_orc_ip_table;
129 static struct orc_entry *g_orc_table;
130
131 static pthread_t orc_sort_thread;
132
orc_ip(const int * ip)133 static inline unsigned long orc_ip(const int *ip)
134 {
135 return (unsigned long)ip + *ip;
136 }
137
orc_sort_cmp(const void * _a,const void * _b)138 static int orc_sort_cmp(const void *_a, const void *_b)
139 {
140 struct orc_entry *orc_a, *orc_b;
141 const int *a = g_orc_ip_table + *(int *)_a;
142 const int *b = g_orc_ip_table + *(int *)_b;
143 unsigned long a_val = orc_ip(a);
144 unsigned long b_val = orc_ip(b);
145
146 if (a_val > b_val)
147 return 1;
148 if (a_val < b_val)
149 return -1;
150
151 /*
152 * The "weak" section terminator entries need to always be on the left
153 * to ensure the lookup code skips them in favor of real entries.
154 * These terminator entries exist to handle any gaps created by
155 * whitelisted .o files which didn't get objtool generation.
156 */
157 orc_a = g_orc_table + (a - g_orc_ip_table);
158 orc_b = g_orc_table + (b - g_orc_ip_table);
159 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED)
160 return 0;
161 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
162 }
163
sort_orctable(void * arg)164 static void *sort_orctable(void *arg)
165 {
166 int i;
167 int *idxs = NULL;
168 int *tmp_orc_ip_table = NULL;
169 struct orc_entry *tmp_orc_table = NULL;
170 unsigned int *orc_ip_size = (unsigned int *)arg;
171 unsigned int num_entries = *orc_ip_size / sizeof(int);
172 unsigned int orc_size = num_entries * sizeof(struct orc_entry);
173
174 idxs = (int *)malloc(*orc_ip_size);
175 if (!idxs) {
176 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s",
177 strerror(errno));
178 pthread_exit(g_err);
179 }
180
181 tmp_orc_ip_table = (int *)malloc(*orc_ip_size);
182 if (!tmp_orc_ip_table) {
183 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s",
184 strerror(errno));
185 pthread_exit(g_err);
186 }
187
188 tmp_orc_table = (struct orc_entry *)malloc(orc_size);
189 if (!tmp_orc_table) {
190 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s",
191 strerror(errno));
192 pthread_exit(g_err);
193 }
194
195 /* initialize indices array, convert ip_table to absolute address */
196 for (i = 0; i < num_entries; i++) {
197 idxs[i] = i;
198 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int);
199 }
200 memcpy(tmp_orc_table, g_orc_table, orc_size);
201
202 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp);
203
204 for (i = 0; i < num_entries; i++) {
205 if (idxs[i] == i)
206 continue;
207
208 /* convert back to relative address */
209 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int);
210 g_orc_table[i] = tmp_orc_table[idxs[i]];
211 }
212
213 free(idxs);
214 free(tmp_orc_ip_table);
215 free(tmp_orc_table);
216 pthread_exit(NULL);
217 }
218 #endif
219
220 #ifdef MCOUNT_SORT_ENABLED
221
compare_values_64(const void * a,const void * b)222 static int compare_values_64(const void *a, const void *b)
223 {
224 uint64_t av = *(uint64_t *)a;
225 uint64_t bv = *(uint64_t *)b;
226
227 if (av < bv)
228 return -1;
229 return av > bv;
230 }
231
compare_values_32(const void * a,const void * b)232 static int compare_values_32(const void *a, const void *b)
233 {
234 uint32_t av = *(uint32_t *)a;
235 uint32_t bv = *(uint32_t *)b;
236
237 if (av < bv)
238 return -1;
239 return av > bv;
240 }
241
242 static int (*compare_values)(const void *a, const void *b);
243
244 /* Only used for sorting mcount table */
rela_write_addend(Elf_Rela * rela,uint64_t val)245 static void rela_write_addend(Elf_Rela *rela, uint64_t val)
246 {
247 elf_parser.rela_write_addend(rela, val);
248 }
249
250 struct func_info {
251 uint64_t addr;
252 uint64_t size;
253 };
254
255 /* List of functions created by: nm -S vmlinux */
256 static struct func_info *function_list;
257 static int function_list_size;
258
259 /* Allocate functions in 1k blocks */
260 #define FUNC_BLK_SIZE 1024
261 #define FUNC_BLK_MASK (FUNC_BLK_SIZE - 1)
262
add_field(uint64_t addr,uint64_t size)263 static int add_field(uint64_t addr, uint64_t size)
264 {
265 struct func_info *fi;
266 int fsize = function_list_size;
267
268 if (!(fsize & FUNC_BLK_MASK)) {
269 fsize += FUNC_BLK_SIZE;
270 fi = realloc(function_list, fsize * sizeof(struct func_info));
271 if (!fi)
272 return -1;
273 function_list = fi;
274 }
275 fi = &function_list[function_list_size++];
276 fi->addr = addr;
277 fi->size = size;
278 return 0;
279 }
280
281 /* Used for when mcount/fentry is before the function entry */
282 static int before_func;
283
284 /* Only return match if the address lies inside the function size */
cmp_func_addr(const void * K,const void * A)285 static int cmp_func_addr(const void *K, const void *A)
286 {
287 uint64_t key = *(const uint64_t *)K;
288 const struct func_info *a = A;
289
290 if (key + before_func < a->addr)
291 return -1;
292 return key >= a->addr + a->size;
293 }
294
295 /* Find the function in function list that is bounded by the function size */
find_func(uint64_t key)296 static int find_func(uint64_t key)
297 {
298 return bsearch(&key, function_list, function_list_size,
299 sizeof(struct func_info), cmp_func_addr) != NULL;
300 }
301
cmp_funcs(const void * A,const void * B)302 static int cmp_funcs(const void *A, const void *B)
303 {
304 const struct func_info *a = A;
305 const struct func_info *b = B;
306
307 if (a->addr < b->addr)
308 return -1;
309 return a->addr > b->addr;
310 }
311
parse_symbols(const char * fname)312 static int parse_symbols(const char *fname)
313 {
314 FILE *fp;
315 char addr_str[20]; /* Only need 17, but round up to next int size */
316 char size_str[20];
317 char type;
318
319 fp = fopen(fname, "r");
320 if (!fp) {
321 perror(fname);
322 return -1;
323 }
324
325 while (fscanf(fp, "%16s %16s %c %*s\n", addr_str, size_str, &type) == 3) {
326 uint64_t addr;
327 uint64_t size;
328
329 /* Only care about functions */
330 if (type != 't' && type != 'T' && type != 'W')
331 continue;
332
333 addr = strtoull(addr_str, NULL, 16);
334 size = strtoull(size_str, NULL, 16);
335 if (add_field(addr, size) < 0)
336 return -1;
337 }
338 fclose(fp);
339
340 qsort(function_list, function_list_size, sizeof(struct func_info), cmp_funcs);
341
342 return 0;
343 }
344
345 static pthread_t mcount_sort_thread;
346 static bool sort_reloc;
347
348 static long rela_type;
349
350 static char m_err[ERRSTR_MAXSZ];
351
352 struct elf_mcount_loc {
353 Elf_Ehdr *ehdr;
354 Elf_Shdr *init_data_sec;
355 uint64_t start_mcount_loc;
356 uint64_t stop_mcount_loc;
357 };
358
359 /* Fill the array with the content of the relocs */
fill_relocs(void * ptr,uint64_t size,Elf_Ehdr * ehdr,uint64_t start_loc)360 static int fill_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc)
361 {
362 Elf_Shdr *shdr_start;
363 Elf_Rela *rel;
364 unsigned int shnum;
365 unsigned int count = 0;
366 int shentsize;
367 void *array_end = ptr + size;
368
369 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
370 shentsize = ehdr_shentsize(ehdr);
371
372 shnum = ehdr_shnum(ehdr);
373 if (shnum == SHN_UNDEF)
374 shnum = shdr_size(shdr_start);
375
376 for (int i = 0; i < shnum; i++) {
377 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
378 void *end;
379
380 if (shdr_type(shdr) != SHT_RELA)
381 continue;
382
383 rel = (void *)ehdr + shdr_offset(shdr);
384 end = (void *)rel + shdr_size(shdr);
385
386 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) {
387 uint64_t offset = rela_offset(rel);
388
389 if (offset >= start_loc && offset < start_loc + size) {
390 if (ptr + long_size > array_end) {
391 snprintf(m_err, ERRSTR_MAXSZ,
392 "Too many relocations");
393 return -1;
394 }
395
396 /* Make sure this has the correct type */
397 if (rela_info(rel) != rela_type) {
398 snprintf(m_err, ERRSTR_MAXSZ,
399 "rela has type %lx but expected %lx\n",
400 (long)rela_info(rel), rela_type);
401 return -1;
402 }
403
404 if (long_size == 4)
405 *(uint32_t *)ptr = rela_addend(rel);
406 else
407 *(uint64_t *)ptr = rela_addend(rel);
408 ptr += long_size;
409 count++;
410 }
411 }
412 }
413 return count;
414 }
415
416 /* Put the sorted vals back into the relocation elements */
replace_relocs(void * ptr,uint64_t size,Elf_Ehdr * ehdr,uint64_t start_loc)417 static void replace_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc)
418 {
419 Elf_Shdr *shdr_start;
420 Elf_Rela *rel;
421 unsigned int shnum;
422 int shentsize;
423
424 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
425 shentsize = ehdr_shentsize(ehdr);
426
427 shnum = ehdr_shnum(ehdr);
428 if (shnum == SHN_UNDEF)
429 shnum = shdr_size(shdr_start);
430
431 for (int i = 0; i < shnum; i++) {
432 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
433 void *end;
434
435 if (shdr_type(shdr) != SHT_RELA)
436 continue;
437
438 rel = (void *)ehdr + shdr_offset(shdr);
439 end = (void *)rel + shdr_size(shdr);
440
441 for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) {
442 uint64_t offset = rela_offset(rel);
443
444 if (offset >= start_loc && offset < start_loc + size) {
445 if (long_size == 4)
446 rela_write_addend(rel, *(uint32_t *)ptr);
447 else
448 rela_write_addend(rel, *(uint64_t *)ptr);
449 ptr += long_size;
450 }
451 }
452 }
453 }
454
fill_addrs(void * ptr,uint64_t size,void * addrs)455 static int fill_addrs(void *ptr, uint64_t size, void *addrs)
456 {
457 void *end = ptr + size;
458 int count = 0;
459
460 for (; ptr < end; ptr += long_size, addrs += long_size, count++) {
461 if (long_size == 4)
462 *(uint32_t *)ptr = elf_parser.r(addrs);
463 else
464 *(uint64_t *)ptr = elf_parser.r8(addrs);
465 }
466 return count;
467 }
468
replace_addrs(void * ptr,uint64_t size,void * addrs)469 static void replace_addrs(void *ptr, uint64_t size, void *addrs)
470 {
471 void *end = ptr + size;
472
473 for (; ptr < end; ptr += long_size, addrs += long_size) {
474 if (long_size == 4)
475 elf_parser.w(*(uint32_t *)ptr, addrs);
476 else
477 elf_parser.w8(*(uint64_t *)ptr, addrs);
478 }
479 }
480
481 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */
sort_mcount_loc(void * arg)482 static void *sort_mcount_loc(void *arg)
483 {
484 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg;
485 uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec)
486 + shdr_offset(emloc->init_data_sec);
487 uint64_t size = emloc->stop_mcount_loc - emloc->start_mcount_loc;
488 unsigned char *start_loc = (void *)emloc->ehdr + offset;
489 Elf_Ehdr *ehdr = emloc->ehdr;
490 void *e_msg = NULL;
491 void *vals;
492 int count;
493
494 vals = malloc(long_size * size);
495 if (!vals) {
496 snprintf(m_err, ERRSTR_MAXSZ, "Failed to allocate sort array");
497 pthread_exit(m_err);
498 }
499
500 if (sort_reloc) {
501 count = fill_relocs(vals, size, ehdr, emloc->start_mcount_loc);
502 /* gcc may use relocs to save the addresses, but clang does not. */
503 if (!count) {
504 count = fill_addrs(vals, size, start_loc);
505 sort_reloc = 0;
506 }
507 } else
508 count = fill_addrs(vals, size, start_loc);
509
510 if (count < 0) {
511 e_msg = m_err;
512 goto out;
513 }
514
515 if (count != size / long_size) {
516 snprintf(m_err, ERRSTR_MAXSZ, "Expected %u mcount elements but found %u\n",
517 (int)(size / long_size), count);
518 e_msg = m_err;
519 goto out;
520 }
521
522 /* zero out any locations not found by function list */
523 if (function_list_size) {
524 for (void *ptr = vals; ptr < vals + size; ptr += long_size) {
525 uint64_t key;
526
527 key = long_size == 4 ? *(uint32_t *)ptr : *(uint64_t *)ptr;
528 if (!find_func(key)) {
529 if (long_size == 4)
530 *(uint32_t *)ptr = 0;
531 else
532 *(uint64_t *)ptr = 0;
533 }
534 }
535 }
536
537 compare_values = long_size == 4 ? compare_values_32 : compare_values_64;
538
539 qsort(vals, count, long_size, compare_values);
540
541 if (sort_reloc)
542 replace_relocs(vals, size, ehdr, emloc->start_mcount_loc);
543 else
544 replace_addrs(vals, size, start_loc);
545
546 out:
547 free(vals);
548
549 pthread_exit(e_msg);
550 }
551
552 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */
get_mcount_loc(struct elf_mcount_loc * emloc,Elf_Shdr * symtab_sec,const char * strtab)553 static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec,
554 const char *strtab)
555 {
556 Elf_Sym *sym, *end_sym;
557 int symentsize = shdr_entsize(symtab_sec);
558 int found = 0;
559
560 sym = (void *)emloc->ehdr + shdr_offset(symtab_sec);
561 end_sym = (void *)sym + shdr_size(symtab_sec);
562
563 while (sym < end_sym) {
564 if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) {
565 emloc->start_mcount_loc = sym_value(sym);
566 if (++found == 2)
567 break;
568 } else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) {
569 emloc->stop_mcount_loc = sym_value(sym);
570 if (++found == 2)
571 break;
572 }
573 sym = (void *)sym + symentsize;
574 }
575
576 if (!emloc->start_mcount_loc) {
577 fprintf(stderr, "get start_mcount_loc error!");
578 return;
579 }
580
581 if (!emloc->stop_mcount_loc) {
582 fprintf(stderr, "get stop_mcount_loc error!");
583 return;
584 }
585 }
586 #else /* MCOUNT_SORT_ENABLED */
parse_symbols(const char * fname)587 static inline int parse_symbols(const char *fname) { return 0; }
588 #endif
589
do_sort(Elf_Ehdr * ehdr,char const * const fname,table_sort_t custom_sort)590 static int do_sort(Elf_Ehdr *ehdr,
591 char const *const fname,
592 table_sort_t custom_sort)
593 {
594 int rc = -1;
595 Elf_Shdr *shdr_start;
596 Elf_Shdr *strtab_sec = NULL;
597 Elf_Shdr *symtab_sec = NULL;
598 Elf_Shdr *extab_sec = NULL;
599 Elf_Shdr *string_sec;
600 Elf_Sym *sym;
601 const Elf_Sym *symtab;
602 Elf32_Word *symtab_shndx = NULL;
603 Elf_Sym *sort_needed_sym = NULL;
604 Elf_Shdr *sort_needed_sec;
605 uint32_t *sort_needed_loc;
606 void *sym_start;
607 void *sym_end;
608 const char *secstrings;
609 const char *strtab;
610 char *extab_image;
611 int sort_need_index;
612 int symentsize;
613 int shentsize;
614 int idx;
615 int i;
616 unsigned int shnum;
617 unsigned int shstrndx;
618 #ifdef MCOUNT_SORT_ENABLED
619 struct elf_mcount_loc mstruct = {0};
620 #endif
621 #ifdef UNWINDER_ORC_ENABLED
622 unsigned int orc_ip_size = 0;
623 unsigned int orc_size = 0;
624 unsigned int orc_num_entries = 0;
625 #endif
626
627 shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
628 shentsize = ehdr_shentsize(ehdr);
629
630 shstrndx = ehdr_shstrndx(ehdr);
631 if (shstrndx == SHN_XINDEX)
632 shstrndx = shdr_link(shdr_start);
633 string_sec = get_index(shdr_start, shentsize, shstrndx);
634 secstrings = (const char *)ehdr + shdr_offset(string_sec);
635
636 shnum = ehdr_shnum(ehdr);
637 if (shnum == SHN_UNDEF)
638 shnum = shdr_size(shdr_start);
639
640 for (i = 0; i < shnum; i++) {
641 Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
642
643 idx = shdr_name(shdr);
644 if (!strcmp(secstrings + idx, "__ex_table"))
645 extab_sec = shdr;
646 if (!strcmp(secstrings + idx, ".symtab"))
647 symtab_sec = shdr;
648 if (!strcmp(secstrings + idx, ".strtab"))
649 strtab_sec = shdr;
650
651 if (shdr_type(shdr) == SHT_SYMTAB_SHNDX)
652 symtab_shndx = (Elf32_Word *)((const char *)ehdr +
653 shdr_offset(shdr));
654
655 #ifdef MCOUNT_SORT_ENABLED
656 /* locate the .init.data section in vmlinux */
657 if (!strcmp(secstrings + idx, ".init.data"))
658 mstruct.init_data_sec = shdr;
659 #endif
660
661 #ifdef UNWINDER_ORC_ENABLED
662 /* locate the ORC unwind tables */
663 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) {
664 orc_ip_size = shdr_size(shdr);
665 g_orc_ip_table = (int *)((void *)ehdr +
666 shdr_offset(shdr));
667 }
668 if (!strcmp(secstrings + idx, ".orc_unwind")) {
669 orc_size = shdr_size(shdr);
670 g_orc_table = (struct orc_entry *)((void *)ehdr +
671 shdr_offset(shdr));
672 }
673 #endif
674 } /* for loop */
675
676 #ifdef UNWINDER_ORC_ENABLED
677 if (!g_orc_ip_table || !g_orc_table) {
678 fprintf(stderr,
679 "incomplete ORC unwind tables in file: %s\n", fname);
680 goto out;
681 }
682
683 orc_num_entries = orc_ip_size / sizeof(int);
684 if (orc_ip_size % sizeof(int) != 0 ||
685 orc_size % sizeof(struct orc_entry) != 0 ||
686 orc_num_entries != orc_size / sizeof(struct orc_entry)) {
687 fprintf(stderr,
688 "inconsistent ORC unwind table entries in file: %s\n",
689 fname);
690 goto out;
691 }
692
693 /* create thread to sort ORC unwind tables concurrently */
694 if (pthread_create(&orc_sort_thread, NULL,
695 sort_orctable, &orc_ip_size)) {
696 fprintf(stderr,
697 "pthread_create orc_sort_thread failed '%s': %s\n",
698 strerror(errno), fname);
699 goto out;
700 }
701 #endif
702 if (!extab_sec) {
703 fprintf(stderr, "no __ex_table in file: %s\n", fname);
704 goto out;
705 }
706
707 if (!symtab_sec) {
708 fprintf(stderr, "no .symtab in file: %s\n", fname);
709 goto out;
710 }
711
712 if (!strtab_sec) {
713 fprintf(stderr, "no .strtab in file: %s\n", fname);
714 goto out;
715 }
716
717 extab_image = (void *)ehdr + shdr_offset(extab_sec);
718 strtab = (const char *)ehdr + shdr_offset(strtab_sec);
719 symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec));
720
721 #ifdef MCOUNT_SORT_ENABLED
722 mstruct.ehdr = ehdr;
723 get_mcount_loc(&mstruct, symtab_sec, strtab);
724
725 if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) {
726 fprintf(stderr,
727 "incomplete mcount's sort in file: %s\n",
728 fname);
729 goto out;
730 }
731
732 /* create thread to sort mcount_loc concurrently */
733 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) {
734 fprintf(stderr,
735 "pthread_create mcount_sort_thread failed '%s': %s\n",
736 strerror(errno), fname);
737 goto out;
738 }
739 #endif
740
741 if (custom_sort) {
742 custom_sort(extab_image, shdr_size(extab_sec));
743 } else {
744 int num_entries = shdr_size(extab_sec) / extable_ent_size;
745 qsort(extab_image, num_entries,
746 extable_ent_size, compare_extable);
747 }
748
749 /* find the flag main_extable_sort_needed */
750 sym_start = (void *)ehdr + shdr_offset(symtab_sec);
751 sym_end = sym_start + shdr_size(symtab_sec);
752 symentsize = shdr_entsize(symtab_sec);
753
754 for (sym = sym_start; (void *)sym + symentsize < sym_end;
755 sym = (void *)sym + symentsize) {
756 if (sym_type(sym) != STT_OBJECT)
757 continue;
758 if (!strcmp(strtab + sym_name(sym),
759 "main_extable_sort_needed")) {
760 sort_needed_sym = sym;
761 break;
762 }
763 }
764
765 if (!sort_needed_sym) {
766 fprintf(stderr,
767 "no main_extable_sort_needed symbol in file: %s\n",
768 fname);
769 goto out;
770 }
771
772 sort_need_index = get_secindex(sym_shndx(sym),
773 ((void *)sort_needed_sym - (void *)symtab) / symentsize,
774 symtab_shndx);
775 sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index);
776 sort_needed_loc = (void *)ehdr +
777 shdr_offset(sort_needed_sec) +
778 sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec);
779
780 /* extable has been sorted, clear the flag */
781 elf_parser.w(0, sort_needed_loc);
782 rc = 0;
783
784 out:
785 #ifdef UNWINDER_ORC_ENABLED
786 if (orc_sort_thread) {
787 void *retval = NULL;
788 /* wait for ORC tables sort done */
789 rc = pthread_join(orc_sort_thread, &retval);
790 if (rc) {
791 fprintf(stderr,
792 "pthread_join failed '%s': %s\n",
793 strerror(errno), fname);
794 } else if (retval) {
795 rc = -1;
796 fprintf(stderr,
797 "failed to sort ORC tables '%s': %s\n",
798 (char *)retval, fname);
799 }
800 }
801 #endif
802
803 #ifdef MCOUNT_SORT_ENABLED
804 if (mcount_sort_thread) {
805 void *retval = NULL;
806 /* wait for mcount sort done */
807 rc = pthread_join(mcount_sort_thread, &retval);
808 if (rc) {
809 fprintf(stderr,
810 "pthread_join failed '%s': %s\n",
811 strerror(errno), fname);
812 } else if (retval) {
813 rc = -1;
814 fprintf(stderr,
815 "failed to sort mcount '%s': %s\n",
816 (char *)retval, fname);
817 }
818 }
819 #endif
820 return rc;
821 }
822
compare_relative_table(const void * a,const void * b)823 static int compare_relative_table(const void *a, const void *b)
824 {
825 int32_t av = (int32_t)elf_parser.r(a);
826 int32_t bv = (int32_t)elf_parser.r(b);
827
828 if (av < bv)
829 return -1;
830 if (av > bv)
831 return 1;
832 return 0;
833 }
834
sort_relative_table(char * extab_image,int image_size)835 static void sort_relative_table(char *extab_image, int image_size)
836 {
837 int i = 0;
838
839 /*
840 * Do the same thing the runtime sort does, first normalize to
841 * being relative to the start of the section.
842 */
843 while (i < image_size) {
844 uint32_t *loc = (uint32_t *)(extab_image + i);
845 elf_parser.w(elf_parser.r(loc) + i, loc);
846 i += 4;
847 }
848
849 qsort(extab_image, image_size / 8, 8, compare_relative_table);
850
851 /* Now denormalize. */
852 i = 0;
853 while (i < image_size) {
854 uint32_t *loc = (uint32_t *)(extab_image + i);
855 elf_parser.w(elf_parser.r(loc) - i, loc);
856 i += 4;
857 }
858 }
859
sort_relative_table_with_data(char * extab_image,int image_size)860 static void sort_relative_table_with_data(char *extab_image, int image_size)
861 {
862 int i = 0;
863
864 while (i < image_size) {
865 uint32_t *loc = (uint32_t *)(extab_image + i);
866
867 elf_parser.w(elf_parser.r(loc) + i, loc);
868 elf_parser.w(elf_parser.r(loc + 1) + i + 4, loc + 1);
869 /* Don't touch the fixup type or data */
870
871 i += sizeof(uint32_t) * 3;
872 }
873
874 qsort(extab_image, image_size / 12, 12, compare_relative_table);
875
876 i = 0;
877 while (i < image_size) {
878 uint32_t *loc = (uint32_t *)(extab_image + i);
879
880 elf_parser.w(elf_parser.r(loc) - i, loc);
881 elf_parser.w(elf_parser.r(loc + 1) - (i + 4), loc + 1);
882 /* Don't touch the fixup type or data */
883
884 i += sizeof(uint32_t) * 3;
885 }
886 }
887
do_file(char const * const fname,void * addr)888 static int do_file(char const *const fname, void *addr)
889 {
890 Elf_Ehdr *ehdr = addr;
891 table_sort_t custom_sort = NULL;
892
893 switch (elf_map_machine(ehdr)) {
894 case EM_AARCH64:
895 #ifdef MCOUNT_SORT_ENABLED
896 sort_reloc = true;
897 rela_type = 0x403;
898 /* arm64 uses patchable function entry placing before function */
899 before_func = 8;
900 #endif
901 /* fallthrough */
902 case EM_386:
903 case EM_LOONGARCH:
904 case EM_RISCV:
905 case EM_S390:
906 case EM_X86_64:
907 custom_sort = sort_relative_table_with_data;
908 break;
909 case EM_PARISC:
910 case EM_PPC:
911 case EM_PPC64:
912 custom_sort = sort_relative_table;
913 break;
914 case EM_ARCOMPACT:
915 case EM_ARCV2:
916 case EM_ARM:
917 case EM_MICROBLAZE:
918 case EM_MIPS:
919 case EM_XTENSA:
920 break;
921 default:
922 fprintf(stderr, "unrecognized e_machine %d %s\n",
923 elf_parser.r2(&ehdr->e32.e_machine), fname);
924 return -1;
925 }
926
927 switch (elf_map_long_size(addr)) {
928 case 4:
929 compare_extable = compare_extable_32,
930 long_size = 4;
931 extable_ent_size = 8;
932
933 if (elf_parser.r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) ||
934 elf_parser.r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) {
935 fprintf(stderr,
936 "unrecognized ET_EXEC/ET_DYN file: %s\n", fname);
937 return -1;
938 }
939
940 break;
941 case 8:
942 compare_extable = compare_extable_64,
943 long_size = 8;
944 extable_ent_size = 16;
945
946 if (elf_parser.r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) ||
947 elf_parser.r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) {
948 fprintf(stderr,
949 "unrecognized ET_EXEC/ET_DYN file: %s\n",
950 fname);
951 return -1;
952 }
953
954 break;
955 default:
956 fprintf(stderr, "unrecognized ELF class %d %s\n",
957 ehdr->e32.e_ident[EI_CLASS], fname);
958 return -1;
959 }
960
961 return do_sort(ehdr, fname, custom_sort);
962 }
963
main(int argc,char * argv[])964 int main(int argc, char *argv[])
965 {
966 int i, n_error = 0; /* gcc-4.3.0 false positive complaint */
967 size_t size = 0;
968 void *addr = NULL;
969 int c;
970
971 while ((c = getopt(argc, argv, "s:")) >= 0) {
972 switch (c) {
973 case 's':
974 if (parse_symbols(optarg) < 0) {
975 fprintf(stderr, "Could not parse %s\n", optarg);
976 return -1;
977 }
978 break;
979 default:
980 fprintf(stderr, "usage: sorttable [-s nm-file] vmlinux...\n");
981 return 0;
982 }
983 }
984
985 if ((argc - optind) < 1) {
986 fprintf(stderr, "usage: sorttable vmlinux...\n");
987 return 0;
988 }
989
990 /* Process each file in turn, allowing deep failure. */
991 for (i = optind; i < argc; i++) {
992 addr = elf_map(argv[i], &size, (1 << ET_EXEC) | (1 << ET_DYN));
993 if (!addr) {
994 ++n_error;
995 continue;
996 }
997
998 if (do_file(argv[i], addr))
999 ++n_error;
1000
1001 elf_unmap(addr, size);
1002 }
1003
1004 return !!n_error;
1005 }
1006